diff --git a/README/ReleaseNotes/v644/index.md b/README/ReleaseNotes/v644/index.md index dde65790e5f56..34228af455f65 100644 --- a/README/ReleaseNotes/v644/index.md +++ b/README/ReleaseNotes/v644/index.md @@ -52,6 +52,51 @@ RenderCore, the WebGL engine of the REve client, is fetched at build time as the ## RooFit +### HistFactory models with fewer RooFit objects + +HistFactory now builds each channel as a `RooHistFactoryPdf`, which corresponds +to the `histfactory_dist` of the HS3 standard. Each sample of a channel is a +single `RooHistFactorySample` object, which holds the nominal histogram and all +its modifiers (NormFactor, OverallSys, HistoSys, StatError, ShapeSys and +ShapeFactor). Before, HistFactory created several RooFit objects for each +histogram (`RooHistFunc`, `RooDataHist`, `PiecewiseInterpolation`, +`FlexibleInterpVar`, `ParamHistFunc` and `RooProduct`). For models with many +systematic variations, this reduces the size of the workspaces in memory and on +disk. The histogram data of a sample is shared between its copies, so cloning +the model, e.g. when creating the likelihood, doesn't copy the histograms. + +The models have the same parameters, constraint terms, observables and datasets +as before, and they give the same values up to floating point rounding. The +binned likelihood optimization, +the code generation backend and the plotting of single samples with +`RooFit::Components("__shapes")` work as before. Code that +navigates the internal structure of HistFactory models, e.g. to retrieve the +`RooHistFunc` of a sample, has to be adapted. The modifiers of a sample are +available via `RooHistFactorySample::data()`. + +When a workspace is written to a ROOT file with `RooWorkspace::Write()` or +`RooWorkspace::writeToFile()`, the histogram data of all its samples goes to an +RNTuple next to the workspace, named after the workspace with the suffix +`_histfactory_arrays`. Like this, HistFactory workspaces are no longer limited +by the 1 GB size limit of single objects in ROOT files. `hist2workspace` writes +its workspaces like this. With `TDirectory::WriteObject()`, the histograms stay +in the workspace object. + +The `Measurement` that HistFactory writes next to the workspaces holds all +histograms as well, so it is subject to the same limit. Since the workspaces +don't need it, writing it can be disabled with the new `writeMeasurement` +member of `RooStats::HistFactory::HistoToWorkspaceFactoryFast::Configuration`. + +The HS3 importer also creates `RooHistFactoryPdf` objects for the +`histfactory_dist` distributions, and the exporter writes them with the names +of their modifiers. + +To get the previous model structure, set the `useHistFactoryPdf` member of the +`RooStats::HistFactory::HistoToWorkspaceFactoryFast::Configuration` to `false`, +use the `-legacy_model_structure` command line argument of `hist2workspace`, or +set `RooJSONFactoryWSTool::config().useHistFactoryPdf = false` for the HS3 +importer. Workspaces with the previous structure can still be read. + ### Removal of the legacy evaluation backend The `legacy` evaluation backend for likelihood and chi-square fits is removed. diff --git a/roofit/codegen/inc/RooFit/CodegenImpl.h b/roofit/codegen/inc/RooFit/CodegenImpl.h index eca7bad066cb2..b954201a51317 100644 --- a/roofit/codegen/inc/RooFit/CodegenImpl.h +++ b/roofit/codegen/inc/RooFit/CodegenImpl.h @@ -42,6 +42,7 @@ class RooFunctorPdfBinding; class RooGamma; class RooGaussian; class RooGenericPdf; +class RooHistFactorySample; class RooHistFunc; class RooHistPdf; class RooLandau; @@ -109,6 +110,7 @@ void codegenImpl(RooFunctorPdfBinding &arg, CodegenContext &ctx); void codegenImpl(RooGamma &arg, CodegenContext &ctx); void codegenImpl(RooGaussian &arg, CodegenContext &ctx); void codegenImpl(RooGenericPdf &arg, CodegenContext &ctx); +void codegenImpl(RooHistFactorySample &arg, CodegenContext &ctx); void codegenImpl(RooHistFunc &arg, CodegenContext &ctx); void codegenImpl(RooHistPdf &arg, CodegenContext &ctx); void codegenImpl(RooLandau &arg, CodegenContext &ctx); @@ -159,6 +161,7 @@ std::string codegenIntegralImpl(RooEfficiency &arg, int code, const char *rangeN std::string codegenIntegralImpl(RooExponential &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooGamma &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooGaussian &arg, int code, const char *rangeName, CodegenContext &ctx); +std::string codegenIntegralImpl(RooHistFactorySample &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooHistFunc &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooHistPdf &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooLandau &arg, int code, const char *rangeName, CodegenContext &ctx); diff --git a/roofit/codegen/src/CodegenImpl.cxx b/roofit/codegen/src/CodegenImpl.cxx index 1150f1ab5b2de..6b13889d0abd9 100644 --- a/roofit/codegen/src/CodegenImpl.cxx +++ b/roofit/codegen/src/CodegenImpl.cxx @@ -59,6 +59,7 @@ #include #include #include +#include #include #include @@ -594,6 +595,108 @@ void codegenImpl(RooGenericPdf &arg, CodegenContext &ctx) ctx.addResult(&arg, funcName + "(" + inputVar + ")"); } +namespace { + +/// The array arguments of the kernels of a RooHistFactorySample. +std::string intArrayArg(CodegenContext &ctx, std::vector const &arr) +{ + return arr.empty() ? "nullptr" : ctx.buildArg(std::span{arr}); +} + +std::string doubleArrayArg(CodegenContext &ctx, std::vector const &arr) +{ + return ctx.buildArg(std::span{arr}); +} + +/// The arguments of RooFit::Detail::MathFuncs::histFactoryBin() and histFactoryIntegral() after the bin. +std::string histFactoryBinArgs(RooHistFactorySample const &arg, CodegenContext &ctx) +{ + auto const &data = arg.data(); + return std::to_string(data.numBins()) + ", " + doubleArrayArg(ctx, data.contents) + ", " + + std::to_string(data.histoSysParams.size()) + ", " + intArrayArg(ctx, data.histoSysParams) + ", " + + intArrayArg(ctx, data.histoSysCodes) + ", " + doubleArrayArg(ctx, data.histoSysLo) + ", " + + doubleArrayArg(ctx, data.histoSysHi) + ", " + std::to_string(data.binFactorNames.size()) + ", " + + intArrayArg(ctx, data.binFactorParams); +} + +/// The scale factor of a RooHistFactorySample, which only depends on the parameters and is therefore computed outside +/// of the loop over the data. +std::string histFactoryScale(RooHistFactorySample const &arg, std::string const ¶ms, CodegenContext &ctx) +{ + auto const &data = arg.data(); + std::string scaleName = ctx.getTmpVarName(); + std::string call = "RooFit::Detail::MathFuncs::histFactoryScale(" + params + ", " + + std::to_string(data.normSysParams.size()) + ", " + intArrayArg(ctx, data.normSysParams) + ", " + + intArrayArg(ctx, data.normSysCodes) + ", " + doubleArrayArg(ctx, data.normSysLo) + ", " + + doubleArrayArg(ctx, data.normSysHi) + ", " + std::to_string(data.normFactorParams.size()) + ", " + + intArrayArg(ctx, data.normFactorParams) + ")"; + ctx.addToCodeBody("double " + scaleName + " = " + call + ";\n", /*isScopeIndep=*/true); + return scaleName; +} + +} // namespace + +void codegenImpl(RooHistFactorySample &arg, CodegenContext &ctx) +{ + auto const &data = arg.data(); + std::string const ¶ms = ctx.buildArg(arg.parameters()); + const std::string scale = arg.parameters().empty() ? "1.0" : histFactoryScale(arg, params, ctx); + + // The index of the bin with the current values of the observables. The last + // observable varies fastest. + std::string idx; + unsigned int stride = 1; + for (int iDim = static_cast(data.edges.size()) - 1; iDim >= 0; --iDim) { + std::vector const &edges = data.edges[iDim]; + const unsigned int nBinsDim = edges.size() - 1; + RooAbsArg const &obs = arg.observables()[iDim]; + // The bin numbers are computed with a coefficient of one and only then + // multiplied with the stride, because uniformBinNumber() would + // multiply with the coefficient before rounding down. + std::string term = + data.uniform[iDim] + ? ctx.buildCall(mathFunc("uniformBinNumber"), edges.front(), edges.back(), obs, nBinsDim, 1.0) + : ctx.buildCall(mathFunc("binNumber"), obs, 1.0, std::span{edges}, edges.size(), nBinsDim - 1, + 0); + if (stride > 1) { + term = std::to_string(stride) + " * " + term; + } + idx = idx.empty() ? term : term + " + " + idx; + stride *= nBinsDim; + } + std::string idxName = ctx.getTmpVarName(); + ctx.addToCodeBody(&arg, "unsigned int " + idxName + " = " + idx + ";\n"); + + std::string value = mathFunc("histFactoryBin") + "(" + (arg.parameters().empty() ? "nullptr" : params) + ", " + + idxName + ", " + histFactoryBinArgs(arg, ctx) + ")"; + if (!arg.yieldsMode()) { + std::vector invVolumes(data.numBins()); + for (std::size_t i = 0; i < invVolumes.size(); ++i) { + invVolumes[i] = 1. / arg.binVolume(i); + } + value = "(" + value + " * (" + doubleArrayArg(ctx, invVolumes) + ")[" + idxName + "])"; + } + ctx.addResult(&arg, value + " * " + scale); +} + +std::string codegenIntegralImpl(RooHistFactorySample &arg, int code, const char *rangeName, CodegenContext &ctx) +{ + // Only integrals over all observables are supported, because otherwise + // the integral depends on the bins of the other observables. + if (code != (1 << arg.observables().size())) { + std::stringstream errorMsg; + errorMsg << "codegen: integrals of RooHistFactorySample \"" << arg.GetName() + << "\" over only some of its observables are not supported"; + oocoutE(&arg, InputArguments) << errorMsg.str() << std::endl; + throw std::runtime_error(errorMsg.str()); + } + std::string const ¶ms = ctx.buildArg(arg.parameters()); + const std::string scale = arg.parameters().empty() ? "1.0" : histFactoryScale(arg, params, ctx); + return "(" + mathFunc("histFactoryIntegral") + "(" + (arg.parameters().empty() ? "nullptr" : params) + ", " + + doubleArrayArg(ctx, arg.integralWeights(code, rangeName)) + ", " + histFactoryBinArgs(arg, ctx) + ") * " + + scale + ")"; +} + void codegenImpl(RooHistFunc &arg, CodegenContext &ctx) { rooHistTranslateImpl(arg, ctx, arg.getInterpolationOrder(), arg.dataHist(), arg.variables(), false, diff --git a/roofit/histfactory/CMakeLists.txt b/roofit/histfactory/CMakeLists.txt index 12794897e76ed..1308f7249107a 100644 --- a/roofit/histfactory/CMakeLists.txt +++ b/roofit/histfactory/CMakeLists.txt @@ -35,6 +35,8 @@ set(HISTFACTORY_HEADERS RooStats/HistFactory/ParamHistFunc.h RooStats/HistFactory/PiecewiseInterpolation.h RooStats/HistFactory/RooBarlowBeestonLL.h + RooStats/HistFactory/RooHistFactoryPdf.h + RooStats/HistFactory/RooHistFactorySample.h ) ROOT_STANDARD_LIBRARY_PACKAGE(HistFactory @@ -53,6 +55,8 @@ ROOT_STANDARD_LIBRARY_PACKAGE(HistFactory src/ParamHistFunc.cxx src/PiecewiseInterpolation.cxx src/RooBarlowBeestonLL.cxx + src/RooHistFactoryPdf.cxx + src/RooHistFactorySample.cxx ${HISTFACTORY_XML_SOURCES} DICTIONARY_OPTIONS "-writeEmptyRootPCM" @@ -61,6 +65,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(HistFactory DEPENDENCIES RooFit RooFitCore + ROOTNTuple Tree RIO Hist diff --git a/roofit/histfactory/inc/LinkDef.h b/roofit/histfactory/inc/LinkDef.h index d3ca88cb74a9b..06ceb39674f09 100644 --- a/roofit/histfactory/inc/LinkDef.h +++ b/roofit/histfactory/inc/LinkDef.h @@ -9,6 +9,9 @@ #pragma link C++ class PiecewiseInterpolation- ; #pragma link C++ class ParamHistFunc+ ; +#pragma link C++ class RooHistFactoryPdf+ ; +#pragma link C++ class RooHistFactorySample- ; +#pragma link C++ class RooHistFactorySample::Data+ ; #pragma link C++ class RooStats::HistFactory::LinInterpVar+ ; #pragma link C++ class RooStats::HistFactory::FlexibleInterpVar+ ; #pragma link C++ class RooStats::HistFactory::HistoToWorkspaceFactoryFast+ ; diff --git a/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h b/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h index a9ca37eaf6bed..5238a07be886b 100644 --- a/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h +++ b/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h @@ -53,6 +53,13 @@ class HistoToWorkspaceFactoryFast : public TObject { bool createPerRegionWorkspaces = true; /// \brief Control whether errors on the data histograms are stored in the workspace (default `false`). bool storeDataError = false; + /// \brief Build each channel as a RooHistFactoryPdf of RooHistFactorySample objects (default `true`). If + /// `false`, the channels are built as before ROOT 6.44, with several RooFit objects for each histogram. + bool useHistFactoryPdf = true; + /// \brief Control whether the Measurement is written to the ROOT files next to the workspaces (default `true`), + /// together with copies of its histograms. The workspaces don't need it. Like any object in a ROOT file, the + /// Measurement, which holds all histograms, is limited to 1 GB. + bool writeMeasurement = true; }; HistoToWorkspaceFactoryFast() {} diff --git a/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactoryPdf.h b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactoryPdf.h new file mode 100644 index 0000000000000..0e04a0297e184 --- /dev/null +++ b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactoryPdf.h @@ -0,0 +1,39 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooStats_HistFactory_RooHistFactoryPdf_h +#define RooStats_HistFactory_RooHistFactoryPdf_h + +#include + +#include + +class RooHistFactorySample; + +class RooHistFactoryPdf : public RooRealSumPdf { +public: + RooHistFactoryPdf() = default; + RooHistFactoryPdf(const char *name, const char *title, RooArgList const &samples); + RooHistFactoryPdf(const RooHistFactoryPdf &other, const char *name = nullptr); + TObject *clone(const char *newname = nullptr) const override { return new RooHistFactoryPdf(*this, newname); } + + std::size_t numSamples() const { return _funcList.size(); } + RooHistFactorySample const &sample(std::size_t i) const; + RooHistFactorySample const *sample(std::string const &sampleName) const; + RooArgList const &observables() const; + + void printMetaArgs(std::ostream &os) const override; + + ClassDefOverride(RooHistFactoryPdf, 1); +}; + +#endif diff --git a/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactorySample.h b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactorySample.h new file mode 100644 index 0000000000000..d78a9d92a7a7e --- /dev/null +++ b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactorySample.h @@ -0,0 +1,171 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooStats_HistFactory_RooHistFactorySample_h +#define RooStats_HistFactory_RooHistFactorySample_h + +#include +#include + +#include + +#include +#include +#include + +class RooHistFactorySample : public RooAbsReal { +public: + /// The types of modifiers that scale each bin with its own parameter. + enum class BinFactorType { + StatError = 0, + ShapeSys = 1, + ShapeFactor = 2 + }; + + /// Scales the sample with a parameter, or with a function of the parameters (a "custom" modifier in HS3). + struct NormFactor { + std::string name; + RooAbsReal *param = nullptr; + }; + + /// Scales the sample with a factor that is interpolated between `lo` and `hi`. The interpolation codes are the ones + /// of FlexibleInterpVar. + struct NormSys { + std::string name; + RooAbsReal *param = nullptr; + double lo = 1.0; + double hi = 1.0; + int interpCode = 4; + }; + + /// Interpolates the bin contents between the `lo` and `hi` variations. The interpolation codes are the ones of + /// PiecewiseInterpolation. + struct HistoSys { + std::string name; + RooAbsReal *param = nullptr; + std::vector lo; + std::vector hi; + int interpCode = 4; + }; + + /// Scales each bin with its own parameter, given in the order of the bins. + struct BinFactor { + std::string name; + BinFactorType type = BinFactorType::ShapeFactor; + RooArgList params; + }; + + /// Everything that defines a sample. The bins are ordered like in RooDataHist and HS3: the last observable varies + /// fastest. + struct Spec { + std::string sampleName; + std::vector contents; + std::vector errors; ///< Absolute statistical uncertainties, optional. + std::vector normFactors; + std::vector normSys; + std::vector histoSys; + std::vector binFactors; + }; + + /// The immutable data of a sample, shared between the clones of a sample. + struct Data { + std::string sampleName; + std::vector> edges; ///< The bin boundaries of each observable. + std::vector uniform; ///< If the binning of an observable is uniform. + std::vector contents; + std::vector errors; + + std::vector normFactorNames; + std::vector normFactorParams; + + std::vector normSysNames; + std::vector normSysParams; + std::vector normSysLo; + std::vector normSysHi; + std::vector normSysCodes; + + std::vector histoSysNames; + std::vector histoSysParams; + std::vector histoSysCodes; + std::vector histoSysLo; ///< One variation after the other. + std::vector histoSysHi; ///< One variation after the other. + + std::vector binFactorNames; + std::vector binFactorTypes; + std::vector binFactorParams; ///< One modifier after the other, one parameter for each bin. + + std::size_t numBins() const { return contents.size(); } + + ClassDefNV(Data, 1); + }; + + RooHistFactorySample() = default; + RooHistFactorySample(const char *name, const char *title, RooArgList const &observables, Spec const &spec); + RooHistFactorySample(const RooHistFactorySample &other, const char *name = nullptr); + TObject *clone(const char *newname = nullptr) const override { return new RooHistFactorySample(*this, newname); } + + std::string const &sampleName() const { return _data->sampleName; } + RooArgList const &observables() const { return _obs; } + RooArgList const ¶meters() const { return _params; } + Data const &data() const { return *_data; } + std::size_t numBins() const { return _data->numBins(); } + + double binVolume(std::size_t iBin) const; + void binValues(std::span out, bool density) const; + + void setHistoSysInterpCode(RooAbsReal const ¶m, int code); + void setNormSysInterpCode(RooAbsReal const ¶m, int code); + + bool isBinnedDistribution(const RooArgSet &obs) const override; + std::list *binBoundaries(RooAbsRealLValue &obs, double xlo, double xhi) const override; + std::list *plotSamplingHint(RooAbsRealLValue &obs, double xlo, double xhi) const override; + + Int_t getAnalyticalIntegral(RooArgSet &allVars, RooArgSet &analVars, const char *rangeName = nullptr) const override; + double analyticalIntegral(Int_t code, const char *rangeName = nullptr) const override; + std::vector integralWeights(Int_t code, const char *rangeName = nullptr) const; + + std::unique_ptr + compileForNormSet(RooArgSet const &normSet, RooFit::Detail::CompileContext &ctx) const override; + + /// If the values are the yields in the bins, which is the case in the binned likelihood. Otherwise, they are the + /// densities, i.e. the yields divided by the bin volumes. + bool yieldsMode() const { return _yieldsMode; } + + void printMetaArgs(std::ostream &os) const override; + +protected: + double evaluate() const override; + void doEval(RooFit::EvalContext &ctx) const override; + +private: + int observableIndex(RooAbsArg const &obs) const; + std::size_t binIndex(std::span values) const; + double binValue(double const *params, std::size_t iBin) const; + double scaleFactor(double const *params) const; + template + void fillParams(ParamValue &¶mValue) const; + void setInterpCode(std::vector Data::*codes, std::vector Data::*params, RooAbsReal const ¶m, int code, + int maxCode, const char *what); + + RooListProxy _obs; + RooListProxy _params; + std::shared_ptr _data; /// _paramVals; /// _binVals; /// _obsVals; /// #include "RooStats/HistFactory/ParamHistFunc.h" +#include "RooRealSumPdf.h" #include "RooAbsPdf.h" #include "RooArgSet.h" #include "RooArgList.h" @@ -41,10 +42,10 @@ namespace HistFactory{ bool FoundSumPdf=false; std::unique_ptr components{sim_channel->getComponents()}; for (auto *sum_pdf_arg : *components) { - std::string NodeClassName = sum_pdf_arg->ClassName(); - if( NodeClassName == std::string("RooRealSumPdf") ) { + // Also finds a RooHistFactoryPdf, which is a RooRealSumPdf of its samples + if (auto *sumPdf = dynamic_cast(sum_pdf_arg)) { FoundSumPdf=true; - sum_pdf = static_cast(sum_pdf_arg); + sum_pdf = sumPdf; break; } } diff --git a/roofit/histfactory/src/HistFactoryNavigation.cxx b/roofit/histfactory/src/HistFactoryNavigation.cxx index 7ae284bf19932..27ccdf0de4b87 100644 --- a/roofit/histfactory/src/HistFactoryNavigation.cxx +++ b/roofit/histfactory/src/HistFactoryNavigation.cxx @@ -788,10 +788,10 @@ namespace RooStats { // the "unconstrained" pdf node for that channel std::unique_ptr comps{pdf->getComponents()}; for (auto *arg : *comps) { - std::string ClassName = arg->ClassName(); - if( ClassName == "RooRealSumPdf" ) { - fChannelSumNodeMap[ChannelName] = static_cast (arg); - break; + // Also finds a RooHistFactoryPdf, which is a RooRealSumPdf of its samples + if (auto *sumPdf = dynamic_cast(arg)) { + fChannelSumNodeMap[ChannelName] = sumPdf; + break; } } } diff --git a/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx b/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx index 14e284fac6453..e42aad6d00101 100644 --- a/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx +++ b/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx @@ -48,6 +48,8 @@ #include "RooStats/RooStatsUtils.h" #include "RooStats/HistFactory/PiecewiseInterpolation.h" #include "RooStats/HistFactory/ParamHistFunc.h" +#include "RooStats/HistFactory/RooHistFactoryPdf.h" +#include "RooStats/HistFactory/RooHistFactorySample.h" #include "RooStats/AsymptoticCalculator.h" #include "HFMsgService.h" @@ -63,6 +65,7 @@ #include "RooStats/HistFactory/HistFactoryException.h" #include +#include #include #include #include @@ -88,6 +91,49 @@ std::vector histToVector(TH1 const &hist) return out; } +namespace { + +/// The bin contents (or errors) of a histogram in the order of the bins of RooDataHist and HS3: the last axis varies +/// fastest. +std::vector histToBinOrder(TH1 const &hist, bool errors = false) +{ + const int nx = hist.GetNbinsX(); + const int ny = hist.GetDimension() > 1 ? hist.GetNbinsY() : 1; + const int nz = hist.GetDimension() > 2 ? hist.GetNbinsZ() : 1; + std::vector out; + out.reserve(nx * ny * nz); + for (int ix = 1; ix <= nx; ++ix) { + for (int iy = 1; iy <= ny; ++iy) { + for (int iz = 1; iz <= nz; ++iz) { + const int bin = hist.GetBin(ix, iy, iz); + out.push_back(errors ? hist.GetBinError(bin) : hist.GetBinContent(bin)); + } + } + } + return out; +} + +/// ParamHistFunc::createParamSet() lists the parameters with the first axis varying fastest, like the bins of a TH1. +/// This reorders them like the bins of RooDataHist and HS3, where the last axis varies fastest. +RooArgList paramsToBinOrder(RooArgList const ¶ms, RooArgList const &observables) +{ + std::array n{1, 1, 1}; + for (std::size_t i = 0; i < observables.size(); ++i) { + n[i] = static_cast(observables[i]).numBins(); + } + RooArgList out; + for (int ix = 0; ix < n[0]; ++ix) { + for (int iy = 0; iy < n[1]; ++iy) { + for (int iz = 0; iz < n[2]; ++iz) { + out.add(params[ix + n[0] * (iy + n[1] * iz)]); + } + } + } + return out; +} + +} // namespace + // use this order for safety on library loading using namespace RooStats; using std::string, std::vector; @@ -578,20 +624,23 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo return normFactor; } - void HistoToWorkspaceFactoryFast::AddConstraintTerms(RooWorkspace& proto, Measurement & meas, string prefix, - string interpName, - std::vector& systList, - vector& constraintTermNames, - vector& totSystTermNames) { + namespace { - // add variables for all the relative overall uncertainties we expect - totSystTermNames.push_back(prefix); + /// A parameter of a NormSys modifier (OverallSys in HistFactory) with its variations. + struct OverallSysTerm { + std::string name; + RooAbsReal *param = nullptr; + double low = 1.0; + double high = 1.0; + }; - RooArgSet params(prefix.c_str()); - vector lowVec; - vector highVec; + /// Create the parameters of the OverallSys modifiers of a sample and their constraint terms. + std::vector makeOverallSysTerms(RooWorkspace& proto, Measurement & meas, string const& prefix, + std::vector& systList, + vector& constraintTermNames) { + + std::vector terms; - std::map::iterator itconstr; for(unsigned int i = 0; i < systList.size(); ++i) { OverallSys& sys = systList.at(i); @@ -600,14 +649,14 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // case of no systematic (is it possible) if (meas.GetNoSyst().count(sys.GetName()) > 0 ) { - cxcoutI(HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - skip systematic " << sys.GetName() << std::endl; + oocxcoutI(nullptr, HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - skip systematic " << sys.GetName() << std::endl; continue; } // case systematic is a gamma constraint if (meas.GetGammaSyst().count(sys.GetName()) > 0 ) { double relerr = meas.GetGammaSyst().find(sys.GetName() )->second; if (relerr <= 0) { - cxcoutI(HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - zero uncertainty assigned - skip systematic " << sys.GetName() << std::endl; + oocxcoutI(nullptr, HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - zero uncertainty assigned - skip systematic " << sys.GetName() << std::endl; continue; } const double tauVal = 1./(relerr*relerr); @@ -634,8 +683,8 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo const_cast(proto.set("globalObservables"))->add(yvar); // add alphaOfBeta in the list of params to interpolate - params.add(alphaOfBeta); - cxcoutI(HistFactory) << "Added a gamma constraint for " << name << std::endl; + terms.push_back({sys.GetName(), &alphaOfBeta, sys.GetLow(), sys.GetHigh()}); + oocxcoutI(nullptr, HistFactory) << "Added a gamma constraint for " << name << std::endl; } else { @@ -647,7 +696,7 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // check if exists a log-normal constraint if (meas.GetLogNormSyst().count(sys.GetName()) == 0 && meas.GetGammaSyst().count(sys.GetName()) == 0 ) { // just add the alpha for the parameters of the FlexibleInterpVar function - params.add(alpha); + terms.push_back({sys.GetName(), &alpha, sys.GetLow(), sys.GetHigh()}); } // case systematic is a log-normal constraint if (meas.GetLogNormSyst().count(sys.GetName()) > 0 ) { @@ -659,20 +708,206 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo RooArgList{emplace(proto, "tau_" + sys.GetName(), 1. / relerr), emplace(proto, "kappa_" + sys.GetName(), 1. + relerr), alpha}); - cxcoutI(HistFactory) << "Added a log-normal constraint for " << name << std::endl; + oocxcoutI(nullptr, HistFactory) << "Added a log-normal constraint for " << name << std::endl; if (RooMsgService::instance().isActive(nullptr, RooFit::HistFactory, RooFit::DEBUG)) { alphaOfBeta.Print("t"); } - params.add(alphaOfBeta); + terms.push_back({sys.GetName(), &alphaOfBeta, sys.GetLow(), sys.GetHigh()}); } } - // add low/high vectors - lowVec.push_back(sys.GetLow()); - highVec.push_back(sys.GetHigh()); } // end sys loop + return terms; + } + + /// Create the parameters of the NormFactors of a sample, and return their names. + std::vector makeNormFactorParams(RooWorkspace& proto, std::string const& channel, Sample& sample, + bool doRatio) { + std::vector names; + for (NormFactor const& norm : sample.GetNormFactorList()) { + string varname = norm.GetName(); + if (doRatio) { + varname += "_" + channel; + } + // The NormFactor may already exist, e.g. because it was created as a function expression during preprocessing. + if (proto.obj(varname) == nullptr) { + oocxcoutI(nullptr, HistFactory) << "making normFactor: " << norm.GetName() << std::endl; + emplace(proto, varname, norm.GetVal(), norm.GetLow(), norm.GetHigh()); + proto.var(varname)->setError(0); // ensure factor is assigned an initial error, even if its zero + } + if (std::find(names.begin(), names.end(), varname) != names.end()) { + oocxcoutI(nullptr, HistFactory) << " is duplicated for , but only one factor will be included." << std::endl; + continue; + } + names.push_back(varname); + } + return names; + } + + /// Collect everything about a sample for the RooHistFactorySample, creating the parameters and constraint terms of + /// its modifiers like for the models before ROOT 6.44. + RooHistFactorySample::Spec makeSampleSpec(RooWorkspace &proto, Measurement &measurement, + std::string const &channelName, Sample &sample, + RooArgList const &observables, bool doRatio, + std::vector &constraintTermNames, RooArgList &statGammas, + std::map &shapeFactorGammas, + std::map &shapeSysGammas) + { + using BinFactorType = RooHistFactorySample::BinFactorType; + + RooHistFactorySample::Spec spec; + spec.sampleName = sample.GetName(); + + const TH1 *nominal = sample.GetHisto(); + oocxcoutI(nullptr, HistFactory) << "processing hist " << nominal->GetName() << std::endl; + spec.contents = histToBinOrder(*nominal); + + // Same order of factors as in the models before ROOT 6.44: OverallSys, NormFactors, Lumi + for (OverallSysTerm const &term : makeOverallSysTerms(proto, measurement, "alpha_", sample.GetOverallSysList(), + constraintTermNames)) { + // Interpolation code 5 of FlexibleInterpVar: polynomial interpolation and exponential extrapolation + spec.normSys.push_back({term.name, term.param, term.low, term.high, 5}); + } + + for (std::string const &name : makeNormFactorParams(proto, channelName, sample, doRatio)) { + auto *param = dynamic_cast(proto.arg(name)); + if (!param) { + throw hf_exc("The NormFactor \"" + name + "\" of sample \"" + sample.GetName() + "\" is not a real value."); + } + spec.normFactors.push_back({name, param}); + } + + if (!sample.GetHistoSysList().empty()) { + RooArgList interpParams = makeInterpolationParameters(sample.GetHistoSysList(), proto); + for (std::size_t i = 0; i < interpParams.size(); ++i) { + HistoSys const &histoSys = sample.GetHistoSysList()[i]; + const bool isUniform = measurement.GetUniformSyst().count(histoSys.GetName()) > 0; + makeGaussianConstraint(interpParams[i], proto, isUniform, constraintTermNames); + // Interpolation code 4 of PiecewiseInterpolation: polynomial interpolation and linear extrapolation + spec.histoSys.push_back({histoSys.GetName(), &static_cast(interpParams[i]), + histToBinOrder(*histoSys.GetHistoLow()), histToBinOrder(*histoSys.GetHistoHigh()), + 4}); + } + } + + if (sample.GetStatError().GetActivate()) { + if (statGammas.empty()) { + statGammas.add(ParamHistFunc::createParamSet(proto, "gamma_stat_" + channelName, observables, + defaultGammaMin, defaultStatErrorGammaMax)); + } + spec.binFactors.push_back({"staterror", BinFactorType::StatError, paramsToBinOrder(statGammas, observables)}); + } + + for (ShapeFactor &shapeFactor : sample.GetShapeFactorList()) { + oocxcoutI(nullptr, HistFactory) << "Sample: " << sample.GetName() << " in channel: " << channelName + << " to be include a ShapeFactor." << std::endl; + auto found = shapeFactorGammas.find(shapeFactor.GetName()); + if (found == shapeFactorGammas.end()) { + RooArgList gammas = ParamHistFunc::createParamSet(proto, "gamma_" + shapeFactor.GetName(), observables); + for (auto *comp : gammas) { + // If the gamma is subject to a preprocess function, it is a RooAbsReal and + // we don't need to set the initial value. + if (auto var = dynamic_cast(comp)) { + var->setVal(shapeFactor.GetVal()); + var->setMin(shapeFactor.GetLow()); + var->setMax(shapeFactor.GetHigh()); + } + } + // Set an initial shape, if requested + if (TH1 const *initialShape = shapeFactor.GetInitialShape()) { + oocxcoutI(nullptr, HistFactory) << "Setting Shape Factor: " << shapeFactor.GetName() + << " to have initial shape from hist: " << initialShape->GetName() << std::endl; + std::vector values = histToVector(*initialShape); + if (values.size() != gammas.size()) { + throw hf_exc("The initial shape of the ShapeFactor \"" + shapeFactor.GetName() + + "\" has the wrong number of bins."); + } + for (std::size_t i = 0; i < gammas.size(); ++i) { + auto *var = dynamic_cast(&gammas[i]); + if (!var) { + throw hf_exc("Cannot set the initial shape of ShapeFactor \"" + shapeFactor.GetName() + + "\": a parameter is not a RooRealVar."); + } + var->setVal(values[i]); + } + } + // Set the variables constant, if requested + if (shapeFactor.IsConstant()) { + oocxcoutI(nullptr, HistFactory) << "Setting Shape Factor: " << shapeFactor.GetName() << " to be constant" << std::endl; + for (auto *comp : gammas) { + if (auto var = dynamic_cast(comp)) { + var->setConstant(true); + } + } + } + found = shapeFactorGammas.emplace(shapeFactor.GetName(), gammas).first; + } + spec.binFactors.push_back( + {shapeFactor.GetName(), BinFactorType::ShapeFactor, paramsToBinOrder(found->second, observables)}); + } + + for (ShapeSys &shapeSys : sample.GetShapeSysList()) { + oocxcoutI(nullptr, HistFactory) << "Sample: " << sample.GetName() << " in channel: " << channelName + << " to include a ShapeSys." << std::endl; + auto found = shapeSysGammas.find(shapeSys.GetName()); + if (found == shapeSysGammas.end()) { + RooArgList gammas = ParamHistFunc::createParamSet(proto, "gamma_" + shapeSys.GetName(), observables, + defaultGammaMin, defaultShapeSysGammaMax); + found = shapeSysGammas.emplace(shapeSys.GetName(), gammas).first; + } + // The constraint terms, from the relative errors of this sample + auto shapeConstraintsInfo = createGammaConstraints(found->second, histToVector(*shapeSys.GetErrorHist()), + minShapeUncertainty, shapeSys.GetConstraintType()); + for (auto const &term : shapeConstraintsInfo.constraints) { + proto.import(*term, RooFit::RecycleConflictNodes()); + constraintTermNames.emplace_back(term->GetName()); + } + // Add the "observed" value to the list of global observables: + RooArgSet *globalSet = const_cast(proto.set("globalObservables")); + for (RooAbsArg *glob : shapeConstraintsInfo.globalObservables) { + globalSet->add(*proto.var(glob->GetName())); + } + spec.binFactors.push_back( + {shapeSys.GetName(), BinFactorType::ShapeSys, paramsToBinOrder(found->second, observables)}); + } + + // The luminosity, as for the models before ROOT 6.44 + RooAbsArg *lumi = proto.arg("Lumi"); + if (!sample.GetNormalizeByTheory()) { + if (!lumi) { + lumi = &emplace(proto, "Lumi", measurement.GetLumi()); + } else { + static_cast(lumi)->setVal(measurement.GetLumi()); + } + } + spec.normFactors.push_back({"Lumi", &static_cast(*lumi)}); + + return spec; + } + + } // namespace + + void HistoToWorkspaceFactoryFast::AddConstraintTerms(RooWorkspace& proto, Measurement & meas, string prefix, + string interpName, + std::vector& systList, + vector& constraintTermNames, + vector& totSystTermNames) { + + // add variables for all the relative overall uncertainties we expect + totSystTermNames.push_back(prefix); + + RooArgSet params(prefix.c_str()); + vector lowVec; + vector highVec; + for (OverallSysTerm const& term : makeOverallSysTerms(proto, meas, prefix, systList, constraintTermNames)) { + params.add(*term.param); + lowVec.push_back(term.low); + highVec.push_back(term.high); + } + if(!systList.empty()){ // this is epsilon(alpha_j), a piece-wise linear interpolation // LinInterpVar interp( (interpName).c_str(), "", params, 1., lowVec, highVec); @@ -880,6 +1115,48 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo std::vector>> statHistPairs; // const std::string statFuncName = "mc_stat_" + channel_name; + // The absolute statistical uncertainties of a sample. + auto makeStatErrorHist = [&](Sample &sample) { + const TH1 *nominal = sample.GetHisto(); + string UncertName = sample.GetName() + "_" + channel_name + "_StatAbsolUncert"; + std::unique_ptr statErrorHist; + + if( sample.GetStatError().GetErrorHist() == nullptr ) { + // Make the absolute stat error + cxcoutI(HistFactory) << "Making Statistical Uncertainty Hist for " + << " Channel: " << channel_name + << " Sample: " << sample.GetName() + << std::endl; + statErrorHist.reset(MakeAbsolUncertaintyHist( UncertName, nominal)); + } else { + // clone the error histograms because in case the sample has not error hist + // it is created in MakeAbsolUncertainty + // we need later to clean statErrorHist + statErrorHist.reset(static_cast(sample.GetStatError().GetErrorHist()->Clone())); + // We assume the (relative) error is provided. + // We must turn it into an absolute error + // using the nominal histogram + cxcoutI(HistFactory) << "Using external histogram for Stat Errors for " + << "\tChannel: " << channel_name + << "\tSample: " << sample.GetName() + << "\tError Histogram: " << statErrorHist->GetName() << std::endl; + // Multiply the relative stat uncertainty by the + // nominal to get the overall stat uncertainty + statErrorHist->Multiply( nominal ); + statErrorHist->SetName( UncertName.c_str() ); + } + return statErrorHist; + }; + + // For the RooHistFactoryPdf: the samples, and the parameters of the + // modifiers that act on each bin, which samples can share. The parameters + // are in the order of ParamHistFunc::createParamSet(). + std::vector sampleSpecs; + std::vector sampleTitles; + RooArgList statGammas; + std::map shapeFactorGammas; + std::map shapeSysGammas; + ///////////////////////////// // shared parameters // this is ratio of lumi to nominal lumi. We will include relative uncertainty in model @@ -903,6 +1180,26 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // and terms that constrain floating to expectation via uncertainties // GHL: Loop over samples instead, which doesn't contain the data for (Sample& sample : channel.GetSamples()) { + if (fCfg.useHistFactoryPdf) { + RooArgList observables = createObservables(sample.GetHisto(), proto); + if (static_cast(sample.GetHisto()->GetDimension()) != fObsNameVec.size()) { + throw hf_exc("The histogram of sample \"" + sample.GetName() + "\" in channel \"" + channel_name + + "\" has another dimension than the other histograms of the channel."); + } + sampleSpecs.push_back(makeSampleSpec(proto, measurement, channel_name, sample, observables, doRatio, + constraintTermNames, statGammas, shapeFactorGammas, shapeSysGammas)); + RooHistFactorySample::Spec &spec = sampleSpecs.back(); + const TH1 *nominal = sample.GetHisto(); + sampleTitles.emplace_back(strlen(nominal->GetTitle()) > 0 ? nominal->GetTitle() : sample.GetName()); + if (sample.GetStatError().GetActivate()) { + cxcoutI(HistFactory) << "Sample: " << sample.GetName() << " to be included in Stat Error " + << "for channel " << channel_name << std::endl; + statHistPairs.emplace_back(nominal, makeStatErrorHist(sample)); + spec.errors = histToBinOrder(*statHistPairs.back().second); + } + continue; + } + string overallSystName = sample.GetName() + "_" + channel_name + "_epsilon"; string systSourcePrefix = "alpha_"; @@ -989,37 +1286,9 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo << "for channel " << channel_name << std::endl; - string UncertName = sample.GetName() + "_" + channel_name + "_StatAbsolUncert"; - std::unique_ptr statErrorHist; - - if( sample.GetStatError().GetErrorHist() == nullptr ) { - // Make the absolute stat error - cxcoutI(HistFactory) << "Making Statistical Uncertainty Hist for " - << " Channel: " << channel_name - << " Sample: " << sample.GetName() - << std::endl; - statErrorHist.reset(MakeAbsolUncertaintyHist( UncertName, nominal)); - } else { - // clone the error histograms because in case the sample has not error hist - // it is created in MakeAbsolUncertainty - // we need later to clean statErrorHist - statErrorHist.reset(static_cast(sample.GetStatError().GetErrorHist()->Clone())); - // We assume the (relative) error is provided. - // We must turn it into an absolute error - // using the nominal histogram - cxcoutI(HistFactory) << "Using external histogram for Stat Errors for " - << "\tChannel: " << channel_name - << "\tSample: " << sample.GetName() - << "\tError Histogram: " << statErrorHist->GetName() << std::endl; - // Multiply the relative stat uncertainty by the - // nominal to get the overall stat uncertainty - statErrorHist->Multiply( nominal ); - statErrorHist->SetName( UncertName.c_str() ); - } - // Save the nominal and error hists // for the building of constraint terms - statHistPairs.emplace_back(nominal, std::move(statErrorHist)); + statHistPairs.emplace_back(nominal, makeStatErrorHist(sample)); // To do the 'conservative' version, we would need to do some // intervention here. We would probably need to create a different @@ -1278,11 +1547,11 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // Using this TH1* of fractinal stat errors, // create a set of constraint terms: - auto chanStatUncertFunc = static_cast(proto.function( statFuncName )); - cxcoutI(HistFactory) << "About to create Constraint Terms from: " - << chanStatUncertFunc->GetName() - << " params: " << chanStatUncertFunc->paramList() - << std::endl; + RooArgList const &statParams = fCfg.useHistFactoryPdf + ? statGammas + : static_cast(proto.function(statFuncName))->paramList(); + cxcoutI(HistFactory) << "About to create Constraint Terms from: " << statFuncName << " params: " << statParams + << std::endl; // Get the constraint type and the // rel error threshold from the (last) @@ -1300,7 +1569,7 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo double statRelErrorThreshold = channel.GetStatErrorConfig().GetRelErrorThreshold(); auto statConstraintsInfo = createGammaConstraints( - chanStatUncertFunc->paramList(), histToVector(*fracStatError), + statParams, histToVector(*fracStatError), statRelErrorThreshold, statConstraintType); for (auto const& term : statConstraintsInfo.constraints) { @@ -1318,8 +1587,31 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo /////////////////////////////////// // for ith bin calculate totN_i = lumi * sum_j expected_j * syst_j - MakeTotalExpected(proto, channel_name+"_model", - sampleScaleFactors, allSampleHistFuncs); + if (fCfg.useHistFactoryPdf) { + const std::string totName = channel_name + "_model"; + RooArgList observables; + for (std::string const &name : fObsNameVec) { + observables.add(*proto.var(name)); + } + RooArgList samples; + for (std::size_t i = 0; i < sampleSpecs.size(); ++i) { + const std::string name = sampleSpecs[i].sampleName + "_" + channel_name + "_shapes"; + RooHistFactorySample sampleFunc{name.c_str(), sampleTitles[i].c_str(), observables, sampleSpecs[i]}; + proto.import(sampleFunc, RooFit::RecycleConflictNodes()); + samples.add(*proto.function(name)); + } + RooHistFactoryPdf tot{totName.c_str(), totName.c_str(), samples}; + // for mixed generation in RooSimultaneous + tot.setAttribute("GenerateBinned"); + // Enable the binned likelihood optimization + if (fCfg.binnedFitOptimization) { + tot.setAttribute("BinnedLikelihood"); + } + proto.import(tot, RooFit::RecycleConflictNodes()); + } else { + MakeTotalExpected(proto, channel_name+"_model", + sampleScaleFactors, allSampleHistFuncs); + } likelihoodTermNames.push_back(channel_name+"_model"); ////////////////////////////////////// diff --git a/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx b/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx index 39103c613fbfe..047ebfa2dfef1 100644 --- a/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx +++ b/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx @@ -183,15 +183,19 @@ RooStats::HistFactory::MakeModelAndMeasurementFast(RooStats::HistFactory::Measur + ch_name + "_" + rowTitle + "_model.root"; cxcoutIHF << "Opening File to hold channel: " << ChannelFileName << std::endl; std::unique_ptr chanFile{TFile::Open( ChannelFileName.c_str(), "RECREATE" )}; - chanFile->WriteTObject(ws_single.get()); - // Now, write the measurement to the file - // Make a new measurement for only this channel - RooStats::HistFactory::Measurement meas_chan( measurement ); - meas_chan.GetChannels().clear(); - meas_chan.GetChannels().push_back( channel ); - cxcoutIHF << "About to write channel measurement to file" << std::endl; - meas_chan.writeToFile( chanFile.get() ); - cxcoutPHF << "Successfully wrote channel to file" << std::endl; + // RooWorkspace::Write() can store the histograms next to the workspace, see RooHistFactorySample. + chanFile->cd(); + ws_single->Write(); + if (cfg.writeMeasurement) { + // Now, write the measurement to the file + // Make a new measurement for only this channel + RooStats::HistFactory::Measurement meas_chan( measurement ); + meas_chan.GetChannels().clear(); + meas_chan.GetChannels().push_back( channel ); + cxcoutIHF << "About to write channel measurement to file" << std::endl; + meas_chan.writeToFile( chanFile.get() ); + cxcoutPHF << "Successfully wrote channel to file" << std::endl; + } } channel_workspaces.emplace_back(std::move(ws_single)); @@ -223,9 +227,12 @@ RooStats::HistFactory::MakeModelAndMeasurementFast(RooStats::HistFactory::Measur cxcoutEHF << "Error: Failed to open file " << CombinedFileName << std::endl; throw hf_exc(); } - combFile->WriteTObject(ws.get()); - cxcoutPHF << "Writing combined measurement to file: " << CombinedFileName << std::endl; - measurement.writeToFile( combFile.get() ); + combFile->cd(); + ws->Write(); + if (cfg.writeMeasurement) { + cxcoutPHF << "Writing combined measurement to file: " << CombinedFileName << std::endl; + measurement.writeToFile( combFile.get() ); + } } return RooFit::makeOwningPtr(std::move(ws)); diff --git a/roofit/histfactory/src/RooHistFactoryPdf.cxx b/roofit/histfactory/src/RooHistFactoryPdf.cxx new file mode 100644 index 0000000000000..44e5dcba11458 --- /dev/null +++ b/roofit/histfactory/src/RooHistFactoryPdf.cxx @@ -0,0 +1,111 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +/** \class RooHistFactoryPdf +\ingroup HistFactory + +A HistFactory channel: the sum of its samples, which are RooHistFactorySample objects. It corresponds to the +`histfactory_dist` of the HS3 standard, without the constraint terms of the parameters, which are separate pdfs. + +The pdf is a RooRealSumPdf whose coefficients are all one, so it supports the binned likelihood optimization +(attribute `BinnedLikelihood`), binned generation (attribute `GenerateBinned`), and the plotting of single samples +with `RooFit::Components()`. The samples share the observables and their binning. +**/ + +#include + +#include + +#include +#include + +#include + +namespace { + +RooArgList unitCoefficients(std::size_t n) +{ + RooArgList coefs; + for (std::size_t i = 0; i < n; ++i) { + coefs.add(RooFit::RooConst(1.0)); + } + return coefs; +} + +void checkSamples(const char *name, RooArgList const &samples) +{ + if (samples.empty()) { + throw std::invalid_argument(std::string("RooHistFactoryPdf \"") + name + "\": there are no samples"); + } + RooHistFactorySample const *first = nullptr; + for (RooAbsArg *arg : samples) { + auto *sample = dynamic_cast(arg); + if (!sample) { + throw std::invalid_argument(std::string("RooHistFactoryPdf \"") + name + "\": \"" + arg->GetName() + + "\" is not a RooHistFactorySample"); + } + if (!first) { + first = sample; + continue; + } + bool same = + sample->observables().size() == first->observables().size() && sample->data().edges == first->data().edges; + for (std::size_t i = 0; same && i < first->observables().size(); ++i) { + same = &sample->observables()[i] == &first->observables()[i]; + } + if (!same) { + throw std::invalid_argument(std::string("RooHistFactoryPdf \"") + name + "\": the sample \"" + + sample->GetName() + "\" has other observables or bins than the sample \"" + + first->GetName() + "\""); + } + } +} + +} // namespace + +/// Create a HistFactory channel from its samples, which need to have the same observables and bins. +RooHistFactoryPdf::RooHistFactoryPdf(const char *name, const char *title, RooArgList const &samples) + : RooRealSumPdf(name, title, (checkSamples(name, samples), samples), unitCoefficients(samples.size()), true) +{ +} + +RooHistFactoryPdf::RooHistFactoryPdf(const RooHistFactoryPdf &other, const char *name) : RooRealSumPdf(other, name) {} + +RooHistFactorySample const &RooHistFactoryPdf::sample(std::size_t i) const +{ + return static_cast(_funcList[i]); +} + +/// The sample with a given name in the HistFactory model, or nullptr if there is none. +RooHistFactorySample const *RooHistFactoryPdf::sample(std::string const &sampleName) const +{ + for (std::size_t i = 0; i < numSamples(); ++i) { + if (sample(i).sampleName() == sampleName) { + return &sample(i); + } + } + return nullptr; +} + +RooArgList const &RooHistFactoryPdf::observables() const +{ + return sample(0).observables(); +} + +void RooHistFactoryPdf::printMetaArgs(std::ostream &os) const +{ + os << "samples=("; + for (std::size_t i = 0; i < numSamples(); ++i) { + os << (i > 0 ? "," : "") << sample(i).GetName(); + } + os << ") "; +} diff --git a/roofit/histfactory/src/RooHistFactorySample.cxx b/roofit/histfactory/src/RooHistFactorySample.cxx new file mode 100644 index 0000000000000..e45c45b214983 --- /dev/null +++ b/roofit/histfactory/src/RooHistFactorySample.cxx @@ -0,0 +1,718 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +/** \class RooHistFactorySample +\ingroup HistFactory + +One sample of a HistFactory channel, i.e. of a RooHistFactoryPdf: a histogram whose bin contents are modified by +parameters. It corresponds to a sample of the `histfactory_dist` in the HS3 standard. The value in bin \f$ b \f$ is + +\f[ + \nu_b = \left( c_b + \sum_i I_i(\alpha_i; l_{i,b}, c_b, h_{i,b}) \right) + \prod_j \gamma_{j,b} \; \prod_k \eta_k(\theta_k) \prod_m \mu_m , +\f] + +with the nominal contents \f$ c_b \f$, the HistoSys variations \f$ l_{i,b} \f$ and \f$ h_{i,b} \f$, the parameters +\f$ \gamma_{j,b} \f$ of the bin-wise modifiers (staterror, shapesys, shapefactor), the NormSys factors +\f$ \eta_k \f$ and the NormFactors \f$ \mu_m \f$. The interpolations are the ones of PiecewiseInterpolation (HistoSys) +and FlexibleInterpVar (NormSys), and if there is a HistoSys, the interpolated bin contents are clamped at zero. + +The value of the function is the density in the bin of the current observable values, i.e. \f$ \nu_b \f$ divided by +the bin volume. In the binned likelihood, the sample is evaluated as the yield \f$ \nu_b \f$ itself. + +In contrast to the HistFactory models before ROOT 6.44, which consist of several RooFit objects for each histogram, +a sample is a single object. The histogram data is shared between the copies of a sample. + +When a RooWorkspace is written to a TFile with RooWorkspace::Write() or RooWorkspace::writeToFile(), the histogram +data of all its samples is written to an RNTuple in the top directory of the file, named after the workspace with the +suffix `_histfactory_arrays`, and the workspace refers to its entries. Like this, the workspace is not limited by the +1 GB size limit of single objects in ROOT files. When the RNTuple name is taken, for example because the workspace is +written again, a number is appended. The RNTuple stays in the file when the workspace is deleted from it. As for all +RNTuples written to a TFile, the other objects in memory that are attached to the file are written too. In all other +cases, for example with TDirectory::WriteObject() or when a sample is written without a workspace, the data is written +together with the sample. +**/ + +#include + +#include "HistFactoryInterpolationCodeUtils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +int checkedIndex(RooArgList ¶ms, RooAbsReal *param, std::string const &sample, std::string const &modifier) +{ + if (!param) { + throw std::invalid_argument("RooHistFactorySample: the modifier \"" + modifier + "\" of sample \"" + sample + + "\" has no parameter"); + } + int index = params.index(param); + if (index < 0) { + params.add(*param); + index = params.size() - 1; + } + return index; +} + +void checkSize(std::size_t size, std::size_t nBins, std::string const &sample, std::string const &what) +{ + if (size != nBins) { + throw std::invalid_argument("RooHistFactorySample: " + what + " of sample \"" + sample + "\" has " + + std::to_string(size) + " bins instead of " + std::to_string(nBins)); + } +} + +double binWidth(RooHistFactorySample::Data const &data, std::size_t iDim, std::size_t iBin) +{ + std::vector const &edges = data.edges[iDim]; + if (data.uniform[iDim]) { + // The same as RooUniformBinning::binWidth() + return (edges.back() - edges.front()) / (edges.size() - 1); + } + return edges[iBin + 1] - edges[iBin]; +} + +std::size_t binInDim(RooHistFactorySample::Data const &data, std::size_t iDim, double x) +{ + std::vector const &edges = data.edges[iDim]; + const int nBins = edges.size() - 1; + if (data.uniform[iDim]) { + // The same as RooUniformBinning::binNumbers() + const double oneOverW = 1. / ((edges.back() - edges.front()) / nBins); + return x >= edges.back() ? nBins - 1 : std::min(nBins - 1, std::max(0, int((x - edges.front()) * oneOverW))); + } + return RooFit::Detail::MathFuncs::binNumber(x, 1.0, edges.data(), edges.size(), nBins - 1, 0); +} + +} // namespace + +/// Create a sample. +/// \param name The name of the RooFit object. +/// \param title The title of the RooFit object. +/// \param observables The observables, which need to be RooRealVars. Their binning defines the bins of the sample. +/// \param spec The bin contents and the modifiers of the sample. +RooHistFactorySample::RooHistFactorySample(const char *name, const char *title, RooArgList const &observables, + Spec const &spec) + : RooAbsReal(name, title), _obs("!obs", "observables", this), _params("!params", "parameters", this) +{ + auto data = std::make_shared(); + data->sampleName = spec.sampleName; + + std::size_t nBins = 1; + for (RooAbsArg *arg : observables) { + auto *obs = dynamic_cast(arg); + if (!obs) { + throw std::invalid_argument(std::string("RooHistFactorySample: the observable \"") + arg->GetName() + + "\" of sample \"" + spec.sampleName + "\" is not a RooRealVar"); + } + _obs.add(*obs); + RooAbsBinning const &binning = obs->getBinning(); + // Each call to RooBinning::array() reallocates the array. + double const *boundaries = binning.array(); + data->edges.emplace_back(boundaries, boundaries + binning.numBoundaries()); + data->uniform.push_back(binning.isUniform()); + // The array of a uniform binning is computed from the low bound and the bin width, so the high bound has to be + // set explicitly to find the bins exactly like RooUniformBinning. + if (binning.isUniform()) { + data->edges.back().back() = binning.highBound(); + } + nBins *= binning.numBins(); + } + + checkSize(spec.contents.size(), nBins, spec.sampleName, "the nominal histogram"); + data->contents = spec.contents; + if (!spec.errors.empty()) { + checkSize(spec.errors.size(), nBins, spec.sampleName, "the histogram of the statistical uncertainties"); + data->errors = spec.errors; + } + + for (NormFactor const &mod : spec.normFactors) { + data->normFactorNames.push_back(mod.name); + data->normFactorParams.push_back(checkedIndex(_params, mod.param, spec.sampleName, mod.name)); + } + + for (NormSys const &mod : spec.normSys) { + data->normSysNames.push_back(mod.name); + data->normSysParams.push_back(checkedIndex(_params, mod.param, spec.sampleName, mod.name)); + data->normSysLo.push_back(mod.lo); + data->normSysHi.push_back(mod.hi); + data->normSysCodes.push_back(mod.interpCode); + } + + for (HistoSys const &mod : spec.histoSys) { + checkSize(mod.lo.size(), nBins, spec.sampleName, "the low variation of \"" + mod.name + "\""); + checkSize(mod.hi.size(), nBins, spec.sampleName, "the high variation of \"" + mod.name + "\""); + data->histoSysNames.push_back(mod.name); + data->histoSysParams.push_back(checkedIndex(_params, mod.param, spec.sampleName, mod.name)); + data->histoSysCodes.push_back(mod.interpCode); + data->histoSysLo.insert(data->histoSysLo.end(), mod.lo.begin(), mod.lo.end()); + data->histoSysHi.insert(data->histoSysHi.end(), mod.hi.begin(), mod.hi.end()); + } + + for (BinFactor const &mod : spec.binFactors) { + checkSize(mod.params.size(), nBins, spec.sampleName, "the parameters of \"" + mod.name + "\""); + data->binFactorNames.push_back(mod.name); + data->binFactorTypes.push_back(static_cast(mod.type)); + for (RooAbsArg *param : mod.params) { + data->binFactorParams.push_back( + checkedIndex(_params, dynamic_cast(param), spec.sampleName, mod.name)); + } + } + + _data = std::move(data); + + // Validate the interpolation codes like the classes that implement them. + for (std::size_t i = 0; i < _data->normSysCodes.size(); ++i) { + if (_data->normSysCodes[i] < 0 || _data->normSysCodes[i] > 5) { + throw std::invalid_argument("RooHistFactorySample: unknown NormSys interpolation code " + + std::to_string(_data->normSysCodes[i]) + " for \"" + _data->normSysNames[i] + + "\""); + } + } + for (std::size_t i = 0; i < _data->histoSysCodes.size(); ++i) { + if (_data->histoSysCodes[i] < 0 || _data->histoSysCodes[i] > 6 || _data->histoSysCodes[i] == 3) { + throw std::invalid_argument("RooHistFactorySample: unknown HistoSys interpolation code " + + std::to_string(_data->histoSysCodes[i]) + " for \"" + _data->histoSysNames[i] + + "\""); + } + } +} + +RooHistFactorySample::RooHistFactorySample(const RooHistFactorySample &other, const char *name) + : RooAbsReal(other, name), + _obs("!obs", this, other._obs), + _params("!params", this, other._params), + _data(other._data), + _yieldsMode(other._yieldsMode) +{ +} + +/// The volume of a bin, i.e. the product of its widths in all observables. +double RooHistFactorySample::binVolume(std::size_t iBin) const +{ + Data const &data = *_data; + // Like RooDataHist, the bin volume is accumulated starting from the first observable. + double volume = 1.0; + std::size_t stride = data.numBins(); + for (std::size_t iDim = 0; iDim < data.edges.size(); ++iDim) { + stride /= data.edges[iDim].size() - 1; + volume *= binWidth(data, iDim, (iBin / stride) % (data.edges[iDim].size() - 1)); + } + return volume; +} + +/// The scale factor of the whole sample, i.e. the product of the NormSys factors and the NormFactors. +double RooHistFactorySample::scaleFactor(double const *params) const +{ + Data const &data = *_data; + return RooFit::Detail::MathFuncs::histFactoryScale( + params, data.normSysParams.size(), data.normSysParams.data(), data.normSysCodes.data(), data.normSysLo.data(), + data.normSysHi.data(), data.normFactorParams.size(), data.normFactorParams.data()); +} + +/// The value in a bin without the scale factor of the sample. +double RooHistFactorySample::binValue(double const *params, std::size_t iBin) const +{ + Data const &data = *_data; + return RooFit::Detail::MathFuncs::histFactoryBin( + params, iBin, data.numBins(), data.contents.data(), data.histoSysParams.size(), data.histoSysParams.data(), + data.histoSysCodes.data(), data.histoSysLo.data(), data.histoSysHi.data(), data.binFactorNames.size(), + data.binFactorParams.data()); +} + +template +void RooHistFactorySample::fillParams(ParamValue &¶mValue) const +{ + _paramVals.resize(_params.size()); + for (std::size_t i = 0; i < _params.size(); ++i) { + _paramVals[i] = paramValue(static_cast(_params[i])); + } +} + +/// The values of all bins at the current parameter values, either as densities or as yields. +void RooHistFactorySample::binValues(std::span out, bool density) const +{ + if (out.size() != numBins()) { + throw std::invalid_argument("RooHistFactorySample::binValues(): the output has the wrong size"); + } + fillParams([](RooAbsReal const ¶m) { return param.getVal(); }); + const double coef = scaleFactor(_paramVals.data()); + for (std::size_t i = 0; i < out.size(); ++i) { + double value = binValue(_paramVals.data(), i); + if (density) { + value *= 1. / binVolume(i); + } + out[i] = value * coef; + } +} + +std::size_t RooHistFactorySample::binIndex(std::span values) const +{ + Data const &data = *_data; + std::size_t index = 0; + for (std::size_t iDim = 0; iDim < data.edges.size(); ++iDim) { + index = index * (data.edges[iDim].size() - 1) + binInDim(data, iDim, values[iDim]); + } + return index; +} + +double RooHistFactorySample::evaluate() const +{ + fillParams([](RooAbsReal const ¶m) { return param.getVal(); }); + _obsVals.resize(_obs.size()); + for (std::size_t i = 0; i < _obs.size(); ++i) { + _obsVals[i] = static_cast(_obs[i]).getVal(); + } + const std::size_t iBin = binIndex(_obsVals); + double value = binValue(_paramVals.data(), iBin); + if (!_yieldsMode) { + value *= 1. / binVolume(iBin); + } + return value * scaleFactor(_paramVals.data()); +} + +void RooHistFactorySample::doEval(RooFit::EvalContext &ctx) const +{ + std::span output = ctx.output(); + fillParams([&](RooAbsReal const ¶m) { return ctx.at(¶m)[0]; }); + const double coef = scaleFactor(_paramVals.data()); + + std::vector> obsValues; + obsValues.reserve(_obs.size()); + for (RooAbsArg *obs : _obs) { + obsValues.push_back(ctx.at(obs)); + } + + // For many entries, all bins are computed only once. + const std::size_t nBins = numBins(); + const bool allBins = output.size() >= nBins; + if (allBins) { + _binVals.resize(nBins); + for (std::size_t i = 0; i < nBins; ++i) { + double value = binValue(_paramVals.data(), i); + if (!_yieldsMode) { + value *= 1. / binVolume(i); + } + _binVals[i] = value * coef; + } + } + + _obsVals.resize(_obs.size()); + for (std::size_t iEntry = 0; iEntry < output.size(); ++iEntry) { + for (std::size_t i = 0; i < _obs.size(); ++i) { + _obsVals[i] = obsValues[i].size() > 1 ? obsValues[i][iEntry] : obsValues[i][0]; + } + const std::size_t iBin = binIndex(_obsVals); + if (allBins) { + output[iEntry] = _binVals[iBin]; + continue; + } + double value = binValue(_paramVals.data(), iBin); + if (!_yieldsMode) { + value *= 1. / binVolume(iBin); + } + output[iEntry] = value * coef; + } +} + +int RooHistFactorySample::observableIndex(RooAbsArg const &obs) const +{ + for (std::size_t i = 0; i < _obs.size(); ++i) { + if (std::string(_obs[i].GetName()) == obs.GetName()) { + return i; + } + } + return -1; +} + +bool RooHistFactorySample::isBinnedDistribution(const RooArgSet &obs) const +{ + for (RooAbsArg *arg : obs) { + if (observableIndex(*arg) < 0) { + return false; + } + } + return true; +} + +std::list *RooHistFactorySample::binBoundaries(RooAbsRealLValue &obs, double xlo, double xhi) const +{ + const int iDim = observableIndex(obs); + if (iDim < 0) { + return nullptr; + } + // Same as RooHistFunc::binBoundaries() + const double delta = (xhi - xlo) * 1e-8; + auto hint = new std::list; + for (double boundary : _data->edges[iDim]) { + if (boundary > xlo - delta && boundary < xhi + delta) { + hint->push_back(boundary); + } + } + return hint; +} + +std::list *RooHistFactorySample::plotSamplingHint(RooAbsRealLValue &obs, double xlo, double xhi) const +{ + const int iDim = observableIndex(obs); + if (iDim < 0) { + return nullptr; + } + return RooCurve::plotSamplingHintForBinBoundaries(_data->edges[iDim], xlo, xhi); +} + +/// Integrals over any subset of the observables are analytical. The code is one plus the bit mask of the integrated +/// observables. +Int_t RooHistFactorySample::getAnalyticalIntegral(RooArgSet &allVars, RooArgSet &analVars, + const char * /*rangeName*/) const +{ + if (_forceNumInt || _yieldsMode) { + return 0; + } + Int_t code = 0; + for (std::size_t i = 0; i < _obs.size(); ++i) { + if (RooAbsArg *arg = allVars.find(_obs[i])) { + analVars.add(*arg); + code |= 1 << i; + } + } + return code == 0 ? 0 : code + 1; +} + +double RooHistFactorySample::analyticalIntegral(Int_t code, const char *rangeName) const +{ + Data const &data = *_data; + const std::vector weights = integralWeights(code, rangeName); + fillParams([](RooAbsReal const ¶m) { return param.getVal(); }); + const double integral = RooFit::Detail::MathFuncs::histFactoryIntegral( + _paramVals.data(), weights.data(), data.numBins(), data.contents.data(), data.histoSysParams.size(), + data.histoSysParams.data(), data.histoSysCodes.data(), data.histoSysLo.data(), data.histoSysHi.data(), + data.binFactorNames.size(), data.binFactorParams.data()); + return integral * scaleFactor(_paramVals.data()); +} + +/// The weights of the bins in an integral with the code from getAnalyticalIntegral(). For an integrated observable, it +/// is the fraction of the bin in the integration range. For an observable that is not integrated, only the bin of the +/// current value has a weight, which is the inverse of its width because the integral is a density in this observable. +std::vector RooHistFactorySample::integralWeights(Int_t code, const char *rangeName) const +{ + Data const &data = *_data; + const std::size_t nDims = data.edges.size(); + const int mask = code - 1; + + std::vector> weightsPerDim(nDims); + for (std::size_t iDim = 0; iDim < nDims; ++iDim) { + const std::size_t nBinsDim = data.edges[iDim].size() - 1; + auto const &obs = static_cast(_obs[iDim]); + std::vector &w = weightsPerDim[iDim]; + w.assign(nBinsDim, 0.0); + if (!(mask & (1 << iDim))) { + const std::size_t iBin = binInDim(data, iDim, obs.getVal()); + w[iBin] = 1. / binWidth(data, iDim, iBin); + continue; + } + const double lo = obs.getMin(rangeName); + const double hi = obs.getMax(rangeName); + for (std::size_t iBin = 0; iBin < nBinsDim; ++iBin) { + const double binLo = data.edges[iDim][iBin]; + const double binHi = data.edges[iDim][iBin + 1]; + if (binLo >= lo && binHi <= hi) { + w[iBin] = 1.0; + } else if (binHi > lo && binLo < hi) { + w[iBin] = (std::min(hi, binHi) - std::max(lo, binLo)) / binWidth(data, iDim, iBin); + } + } + } + + const std::size_t nBins = data.numBins(); + std::vector weights(nBins); + for (std::size_t iBin = 0; iBin < nBins; ++iBin) { + double weight = 1.0; + std::size_t stride = nBins; + for (std::size_t iDim = 0; iDim < nDims; ++iDim) { + const std::size_t nBinsDim = data.edges[iDim].size() - 1; + stride /= nBinsDim; + weight *= weightsPerDim[iDim][(iBin / stride) % nBinsDim]; + } + weights[iBin] = weight; + } + return weights; +} + +std::unique_ptr +RooHistFactorySample::compileForNormSet(RooArgSet const &normSet, RooFit::Detail::CompileContext &ctx) const +{ + // In the binned likelihood, the values of the pdf are directly the yields + // in the bins, like for a RooBinWidthFunction in that mode. + if (ctx.binnedLikelihoodMode()) { + auto newArg = std::make_unique(*this); + newArg->_yieldsMode = true; + ctx.markAsCompiled(*newArg); + ctx.compileServers(*newArg, {}); + ctx.setBinWidthFuncFlag(true); + return newArg; + } + return RooAbsReal::compileForNormSet(normSet, ctx); +} + +void RooHistFactorySample::setInterpCode(std::vector Data::*codes, std::vector Data::*params, + RooAbsReal const ¶m, int code, int maxCode, const char *what) +{ + // By name, so that the parameter can also come from another copy of the model. + const int iParam = _params.index(param.GetName()); + auto data = std::make_shared(*_data); + bool found = false; + for (std::size_t i = 0; i < (data.get()->*params).size(); ++i) { + if ((data.get()->*params)[i] == iParam) { + RooStats::HistFactory::Detail::setInterpolationCode(*this, what, param, data.get()->*codes, i, code, maxCode); + found = true; + } + } + if (!found) { + coutE(InputArguments) << what << "::setInterpCode ERROR: " << param.GetName() << " is not a parameter of a " + << what << " modifier of sample \"" << sampleName() << "\"" << std::endl; + return; + } + _data = std::move(data); + setValueDirty(); +} + +/// Set the interpolation code of the HistoSys modifiers with the given parameter. The codes are the ones of +/// PiecewiseInterpolation. +void RooHistFactorySample::setHistoSysInterpCode(RooAbsReal const ¶m, int code) +{ + setInterpCode(&Data::histoSysCodes, &Data::histoSysParams, param, code, 6, "HistoSys"); +} + +/// Set the interpolation code of the NormSys modifiers with the given parameter. The codes are the ones of +/// FlexibleInterpVar. +void RooHistFactorySample::setNormSysInterpCode(RooAbsReal const ¶m, int code) +{ + setInterpCode(&Data::normSysCodes, &Data::normSysParams, param, code, 5, "NormSys"); +} + +void RooHistFactorySample::printMetaArgs(std::ostream &os) const +{ + os << "sample=" << sampleName() << " bins=" << numBins(); + auto printNames = [&](const char *label, std::vector const &names) { + if (names.empty()) { + return; + } + os << " " << label << "=("; + for (std::size_t i = 0; i < names.size(); ++i) { + os << (i > 0 ? "," : "") << names[i]; + } + os << ")"; + }; + printNames("normfactors", _data->normFactorNames); + printNames("normsys", _data->normSysNames); + printNames("histosys", _data->histoSysNames); + printNames("binfactors", _data->binFactorNames); + os << " "; +} + +namespace { + +using Data = RooHistFactorySample::Data; +using RooFit::Detail::WorkspaceStreamingContext; + +/// How the data of a sample is stored, written after the members of the sample. +enum StorageType : Int_t { + inlineStorage = 0, + rntupleStorage = 1 +}; + +std::unique_ptr makeModel() +{ + auto model = ROOT::RNTupleModel::CreateBare(); + model->MakeField("data"); + return model; +} + +/// Writes the data of the samples in a workspace to one RNTuple. +class DataWriter : public WorkspaceStreamingContext::Extension { +public: + DataWriter(TFile &file, std::string const &workspaceName) + { + const std::string baseName = workspaceName + "_histfactory_arrays"; + _name = baseName; + for (int i = 1; file.GetKey(_name.c_str()); ++i) { + _name = baseName + "_" + std::to_string(i); + } + ROOT::RNTupleWriteOptions options; + options.SetCompression(file.GetCompressionSettings()); + _writer = ROOT::RNTupleWriter::Append(makeModel(), _name, file, options); + _entry = _writer->CreateRawPtrWriteEntry(); + } + + std::string const &name() const { return _name; } + + /// Write the data, if it was not written already, and return its entry number. + std::uint64_t add(Data const &data) + { + auto found = _entries.find(&data); + if (found != _entries.end()) { + return found->second; + } + _entry->BindRawPtr("data", &data); + _writer->Fill(*_entry); + const std::uint64_t index = _entries.size(); + _entries.emplace(&data, index); + return index; + } + + void finalize() override + { + _writer->CommitDataset(); + _writer.reset(); + } + +private: + std::string _name; + std::unique_ptr _writer; + std::unique_ptr _entry; + std::unordered_map _entries; +}; + +/// Reads the data of the samples from the RNTuples of a file. +class DataReader : public WorkspaceStreamingContext::Extension { +public: + std::shared_ptr get(TFile &file, std::string const &name, std::uint64_t index) + { + auto &source = _sources[name]; + auto found = source.data.find(index); + if (found != source.data.end()) { + return found->second; + } + if (!source.reader) { + // The first cycle is the one that was written with the workspace. Later cycles can come from copying + // the file, e.g. with hadd, which writes the workspace and with it a new RNTuple before copying this one. + std::unique_ptr anchor{file.Get((name + ";1").c_str())}; + if (!anchor) { + throw std::runtime_error("RooHistFactorySample: the RNTuple \"" + name + + "\" with the histogram data is missing in the file \"" + file.GetName() + "\""); + } + source.reader = ROOT::RNTupleReader::Open(makeModel(), *anchor); + source.entry = source.reader->GetModel().CreateBareEntry(); + } + auto data = std::make_shared(); + source.entry->BindRawPtr("data", data.get()); + source.reader->LoadEntry(index, *source.entry); + source.data.emplace(index, data); + return data; + } + +private: + struct Source { + std::unique_ptr reader; + std::unique_ptr entry; + std::unordered_map> data; ///< To share data between samples again. + }; + std::map _sources; +}; + +/// The file of the RNTuple for the data, if the buffer writes a workspace to a file with RooWorkspace::Write(). +TFile *fileForRNTuple(WorkspaceStreamingContext const *context, TBuffer &buffer) +{ + if (!context || !context->file() || buffer.GetParent() != context->file()) { + return nullptr; + } + // Not for TMessage or TBufferJSON, for example. + return buffer.IsA() == TBufferFile::Class() ? context->file() : nullptr; +} + +template +T &getExtension(WorkspaceStreamingContext &context, std::string const &name, Create &&create) +{ + if (auto *extension = context.extension(name)) { + return static_cast(*extension); + } + return static_cast(context.addExtension(name, create())); +} + +} // namespace + +void RooHistFactorySample::Streamer(TBuffer &buffer) +{ + static TClass *dataClass = TClass::GetClass(); + if (buffer.IsReading()) { + buffer.ReadClassBuffer(RooHistFactorySample::Class(), this); + Int_t storage = inlineStorage; + buffer >> storage; + if (storage == inlineStorage) { + auto data = std::make_shared(); + buffer.StreamObject(data.get(), dataClass); + _data = std::move(data); + return; + } + if (storage != rntupleStorage) { + throw std::runtime_error("RooHistFactorySample::Streamer(): unknown storage of the histogram data"); + } + std::string name; + ULong64_t index = 0; + buffer.ReadStdString(&name); + buffer >> index; + auto *file = dynamic_cast(buffer.GetParent()); + if (!file) { + throw std::runtime_error("RooHistFactorySample::Streamer(): the histogram data is in the RNTuple \"" + name + + "\", but the sample is not read from a file"); + } + if (auto *context = WorkspaceStreamingContext::current()) { + auto &reader = getExtension(*context, "RooHistFactorySample::DataReader", + [] { return std::make_unique(); }); + _data = reader.get(*file, name, index); + } else { + _data = DataReader{}.get(*file, name, index); + } + } else { + buffer.WriteClassBuffer(RooHistFactorySample::Class(), this); + auto *context = WorkspaceStreamingContext::current(); + TFile *file = fileForRNTuple(context, buffer); + if (!file) { + buffer << static_cast(inlineStorage); + buffer.StreamObject(const_cast(_data.get()), dataClass); + return; + } + auto &writer = getExtension(*context, "RooHistFactorySample::DataWriter", [&] { + return std::make_unique(*file, context->workspace().GetName()); + }); + const ULong64_t index = writer.add(*_data); + buffer << static_cast(rntupleStorage); + buffer.WriteStdString(&writer.name()); + buffer << index; + } +} diff --git a/roofit/histfactory/src/hist2workspace.cxx b/roofit/histfactory/src/hist2workspace.cxx index c0c1b2b0d0faa..5b2dece969155 100644 --- a/roofit/histfactory/src/hist2workspace.cxx +++ b/roofit/histfactory/src/hist2workspace.cxx @@ -46,11 +46,16 @@ // help="disable the binned fit optimization used in HistFactory since ROOT 6.28", // action="store_true", // ) +// parser.add_argument( +// "-legacy_model_structure", +// help="build the channels from several RooFit objects per histogram, as before ROOT 6.44", +// action="store_true", +// ) // return parser // ``` constexpr static const char kCommandLineOptionsHelp[] = R"RAW( -usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] +usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] [-legacy_model_structure] hist2workspace is a utility to create RooFit/RooStats workspace from histograms @@ -59,6 +64,8 @@ hist2workspace is a utility to create RooFit/RooStats workspace from histograms -v switch HistFactory message stream to INFO level -vv switch HistFactory message stream to DEBUG level -disable_binned_fit_optimization disable the binned fit optimization used in HistFactory since ROOT 6.28 + -legacy_model_structure build the channels from several RooFit objects per histogram, as before + ROOT 6.44 )RAW"; @@ -102,6 +109,7 @@ namespace RooStats { * -v Switch HistFactory message stream to INFO level. * -vv Switch HistFactory message stream to DEBUG level. * -disable_binned_fit_optimization Disable the binned fit optimization used in HistFactory since ROOT 6.28. + * -legacy_model_structure Build the channels from several RooFit objects per histogram, as before ROOT 6.44. */ int main(int argc, char** argv) { @@ -145,6 +153,11 @@ int main(int argc, char** argv) { continue; } + if(input == "-legacy_model_structure") { + cfg.useHistFactoryPdf = false; + continue; + } + driverArg = argv[i]; } diff --git a/roofit/histfactory/test/CMakeLists.txt b/roofit/histfactory/test/CMakeLists.txt index 84e9553995069..ff541aeb57287 100644 --- a/roofit/histfactory/test/CMakeLists.txt +++ b/roofit/histfactory/test/CMakeLists.txt @@ -20,6 +20,11 @@ if(clad) target_compile_definitions(testHistFactory PUBLIC TEST_CODEGEN_AD) endif(clad) +ROOT_ADD_GTEST(testRooHistFactoryPdf testRooHistFactoryPdf.cxx LIBRARIES RooFitCore RooFit RooStats HistFactory) +if(clad) + target_compile_definitions(testRooHistFactoryPdf PUBLIC TEST_CODEGEN_AD) +endif(clad) + ROOT_ADD_GTEST(testParamHistFunc testParamHistFunc.cxx LIBRARIES RooFitCore HistFactory) ROOT_ADD_GTEST(testPiecewiseInterpolation testPiecewiseInterpolation.cxx LIBRARIES RooFitCore HistFactory) ROOT_ADD_GTEST(testHistFactoryPlotting testHistFactoryPlotting.cxx LIBRARIES RooFitCore HistFactory) diff --git a/roofit/histfactory/test/testHistFactory.cxx b/roofit/histfactory/test/testHistFactory.cxx index 9c8e2b804925e..8abe8e1008138 100644 --- a/roofit/histfactory/test/testHistFactory.cxx +++ b/roofit/histfactory/test/testHistFactory.cxx @@ -4,6 +4,8 @@ #include #include +#include +#include #ifdef HISTFACTORY_XML #include #endif @@ -363,7 +365,7 @@ TEST_P(HFFixture, ModelProperties) auto simPdf = dynamic_cast(ws->pdf("simPdf")); ASSERT_NE(simPdf, nullptr); - auto channelPdf = dynamic_cast(ws->pdf("channel1_model")); + auto channelPdf = dynamic_cast(ws->pdf("channel1_model")); ASSERT_NE(channelPdf, nullptr); auto obs = dynamic_cast(ws->var("obs_x_channel1")); @@ -397,36 +399,43 @@ TEST_P(HFFixture, ModelProperties) } } - // Check that sub models depend on their systematic uncertainties. - for (auto &subModelName : std::initializer_list{"signal_channel1_shapes", "background1_channel1_shapes", - "background2_channel1_shapes"}) { - auto subModel = ws->function(subModelName); - ASSERT_NE(subModel, nullptr) << "Unable to retrieve sub model with name " << subModelName; - if (subModelName.find("signal") != std::string::npos) { - EXPECT_FALSE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); - EXPECT_FALSE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); + // Check that the samples depend on their systematic uncertainties. + auto sample = [&](std::string const &name) -> RooHistFactorySample & { + auto *out = dynamic_cast(ws->function((name + "_channel1_shapes").c_str())); + if (!out) { + throw std::runtime_error("Unable to retrieve sample " + name); + } + EXPECT_EQ(out, channelPdf->sample(name)) << name; + return *out; + }; + EXPECT_EQ(channelPdf->numSamples(), 3u); + for (auto &sampleName : std::initializer_list{"signal", "background1", "background2"}) { + RooHistFactorySample &subModel = sample(sampleName); + if (sampleName == "signal") { + EXPECT_FALSE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); + EXPECT_FALSE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); } else { - EXPECT_TRUE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); - EXPECT_TRUE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); + EXPECT_TRUE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); + EXPECT_TRUE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); } } - EXPECT_TRUE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("SigXsecOverSM"))); - EXPECT_FALSE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("SigXsecOverSM"))); - EXPECT_FALSE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("SigXsecOverSM"))); + EXPECT_TRUE(sample("signal").dependsOn(*ws->var("SigXsecOverSM"))); + EXPECT_FALSE(sample("background1").dependsOn(*ws->var("SigXsecOverSM"))); + EXPECT_FALSE(sample("background2").dependsOn(*ws->var("SigXsecOverSM"))); if (makeModelMode == MakeModelMode::OverallSyst) { - EXPECT_FALSE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst2"))); - EXPECT_FALSE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst3"))); - EXPECT_FALSE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst4"))); + EXPECT_FALSE(sample("signal").dependsOn(*ws->var("alpha_syst2"))); + EXPECT_FALSE(sample("signal").dependsOn(*ws->var("alpha_syst3"))); + EXPECT_FALSE(sample("signal").dependsOn(*ws->var("alpha_syst4"))); - EXPECT_TRUE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst2"))); - EXPECT_TRUE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst3"))); - EXPECT_FALSE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst4"))); + EXPECT_TRUE(sample("background1").dependsOn(*ws->var("alpha_syst2"))); + EXPECT_TRUE(sample("background1").dependsOn(*ws->var("alpha_syst3"))); + EXPECT_FALSE(sample("background1").dependsOn(*ws->var("alpha_syst4"))); - EXPECT_FALSE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst2"))); - EXPECT_TRUE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst3"))); - EXPECT_TRUE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst4"))); + EXPECT_FALSE(sample("background2").dependsOn(*ws->var("alpha_syst2"))); + EXPECT_TRUE(sample("background2").dependsOn(*ws->var("alpha_syst3"))); + EXPECT_TRUE(sample("background2").dependsOn(*ws->var("alpha_syst4"))); } EXPECT_EQ(*mc->GetParametersOfInterest()->begin(), ws->var("SigXsecOverSM")); @@ -752,3 +761,57 @@ TEST(HistFactory, ShapeFactorValueAndRange) } } } + +// With HistoToWorkspaceFactoryFast::Configuration::writeMeasurement, the files of the channels and of the combined +// model have the Measurement and the directories with its histograms next to the workspace; without, only the +// workspace. +TEST(HistFactory, WriteMeasurement) +{ + using namespace RooStats::HistFactory; + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + const std::string inputFileName = "TestWriteMeasurement_input.root"; + { + TFile f(inputFileName.c_str(), "RECREATE"); + TH1D data{"data", "data", 2, 1, 2}; + TH1D signal{"signal", "signal", 2, 1, 2}; + data.SetBinContent(1, 12); + data.SetBinContent(2, 21); + signal.SetBinContent(1, 10); + signal.SetBinContent(2, 20); + f.WriteTObject(&data); + f.WriteTObject(&signal); + } + + for (bool writeMeasurement : {true, false}) { + SCOPED_TRACE(writeMeasurement ? "with Measurement" : "without Measurement"); + const std::string prefix = std::string("TestWriteMeasurement_") + (writeMeasurement ? "with" : "without"); + + Measurement meas("meas", "meas"); + meas.SetOutputFilePrefix(prefix); + meas.SetPOI("mu"); + meas.SetLumi(1.0); + meas.SetLumiRelErr(0.1); + Channel chan("channel1"); + chan.SetData("data", inputFileName); + Sample sig("signal", "signal", inputFileName); + sig.AddNormFactor("mu", 1, 0, 3); + chan.AddSample(sig); + meas.AddChannel(chan); + meas.CollectHistograms(); + + HistoToWorkspaceFactoryFast::Configuration cfg; + cfg.writeMeasurement = writeMeasurement; + ASSERT_NE(std::unique_ptr{MakeModelAndMeasurementFast(meas, cfg)}, nullptr); + + for (auto const &[fileSuffix, wsName] : std::vector>{ + {"_channel1_meas_model.root", "channel1"}, {"_combined_meas_model.root", "combined"}}) { + SCOPED_TRACE(fileSuffix); + std::unique_ptr file{TFile::Open((prefix + fileSuffix).c_str())}; + ASSERT_TRUE(file && file->IsOpen()); + EXPECT_NE(std::unique_ptr{file->Get(wsName.c_str())}, nullptr); + EXPECT_EQ(std::unique_ptr{file->Get("meas")} != nullptr, writeMeasurement); + EXPECT_EQ(file->GetKey("channel1_hists") != nullptr, writeMeasurement); + } + } +} diff --git a/roofit/histfactory/test/testRooHistFactoryPdf.cxx b/roofit/histfactory/test/testRooHistFactoryPdf.cxx new file mode 100644 index 0000000000000..d94361e4f1e07 --- /dev/null +++ b/roofit/histfactory/test/testRooHistFactoryPdf.cxx @@ -0,0 +1,435 @@ +// Tests for RooHistFactoryPdf and RooHistFactorySample +// Authors: Jonas Rembser, CERN 2026 + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../../roofitcore/test/gtest_wrapper.h" + +#include +#include + +using namespace RooStats::HistFactory; + +namespace { + +TH1 *makeHist(TRandom3 &rng, const char *name, int dim, double scale) +{ + double xbins[] = {0., 0.5, 1.7, 2.0}; + TH1 *h = dim == 1 ? static_cast(new TH1D(name, name, 3, xbins)) + : static_cast(new TH2D(name, name, 3, xbins, 2, 0, 1)); + h->SetDirectory(nullptr); + for (int i = 1; i <= h->GetNbinsX(); ++i) { + for (int j = 1; j <= h->GetNbinsY(); ++j) { + const int bin = h->GetBin(i, j); + const double v = scale * rng.Uniform(5, 50); + h->SetBinContent(bin, v); + h->SetBinError(bin, std::sqrt(v) * rng.Uniform(0.1, 0.5)); + } + } + return h; +} + +TH1 *varied(TRandom3 &rng, TH1 const *h, const char *name, double lo, double hi) +{ + auto *out = static_cast(h->Clone(name)); + out->SetDirectory(nullptr); + for (int b = 0; b < out->GetNcells(); ++b) { + out->SetBinContent(b, h->GetBinContent(b) * rng.Uniform(lo, hi)); + } + return out; +} + +/// A measurement with all types of modifiers, and with non-uniform bins. +std::unique_ptr makeMeasurement(int dim) +{ + TRandom3 rng{1337}; + auto meas = std::make_unique("meas", "meas"); + meas->SetPOI("mu"); + meas->SetLumi(1.0); + meas->SetLumiRelErr(0.05); + meas->AddLogNormSyst("lognorm", 0.1); + + Channel chan("ch"); + TH1 *data = makeHist(rng, "data", dim, 3.0); + for (int b = 0; b < data->GetNcells(); ++b) { + data->SetBinContent(b, std::round(data->GetBinContent(b))); + } + chan.SetData(data); + chan.SetStatErrorConfig(0.01, "Poisson"); + + Sample sig("signal"); + TH1 *hsig = makeHist(rng, "sig", dim, 1.0); + sig.SetHisto(hsig); + sig.AddNormFactor("mu", 1, 0, 5); + sig.AddOverallSys("syst1", 0.97, 1.03); + sig.AddOverallSys("lognorm", 0.9, 1.1); + for (const char *name : {"shape1", "shape2"}) { + HistoSys hs; + hs.SetName(name); + hs.SetHistoLow(varied(rng, hsig, (std::string{name} + "Lo").c_str(), 0.6, 1.0)); + hs.SetHistoHigh(varied(rng, hsig, (std::string{name} + "Hi").c_str(), 1.0, 1.4)); + sig.AddHistoSys(hs); + } + sig.ActivateStatError(); + chan.AddSample(sig); + + Sample bkg("background"); + TH1 *hbkg = makeHist(rng, "bkg", dim, 2.0); + bkg.SetHisto(hbkg); + bkg.AddOverallSys("syst2", 0.9, 1.2); + bkg.ActivateStatError(); + ShapeSys ss; + ss.SetName("bkgShape"); + TH1 *relErr = varied(rng, hbkg, "relErr", 0.0, 0.0); + for (int b = 0; b < relErr->GetNcells(); ++b) { + relErr->SetBinContent(b, rng.Uniform(0.02, 0.2)); + } + ss.SetErrorHist(relErr); + ss.SetConstraintType(Constraint::Poisson); + bkg.AddShapeSys(ss); + chan.AddSample(bkg); + + Sample free("free"); + free.SetHisto(makeHist(rng, "free", dim, 0.5)); + free.SetNormalizeByTheory(false); + free.AddShapeFactor("freeShape", 1.0, 0.0, 5.0); + chan.AddSample(free); + + meas->AddChannel(chan); + return meas; +} + +std::unique_ptr makeWorkspace(Measurement &meas, bool useHistFactoryPdf) +{ + HistoToWorkspaceFactoryFast::Configuration cfg; + cfg.useHistFactoryPdf = useHistFactoryPdf; + cfg.createPerRegionWorkspaces = false; + return std::unique_ptr{HistoToWorkspaceFactoryFast::MakeCombinedModel(meas, cfg)}; +} + +/// Change the parameters in both workspaces to the same values, close to the current ones. The constrained parameters +/// are changed by at most their uncertainty, so that the constraint terms don't underflow to zero. +void setParameters(TRandom3 &rng, RooWorkspace &ws1, RooWorkspace &ws2) +{ + for (RooAbsArg *arg : ws1.allVars()) { + auto *var1 = static_cast(arg); + auto *var2 = ws2.var(var1->GetName()); + if (var1->isConstant() || !var2) { + continue; + } + const double shift = var1->getError() > 0. ? var1->getError() : 0.3; + const double val = std::clamp(var1->getVal() + rng.Uniform(-shift, shift), var1->getMin(), var1->getMax()); + var1->setVal(val); + var2->setVal(val); + } +} + +/// A workspace with one sample whose data is much larger than the rest of the workspace. +std::unique_ptr makeLargeSampleWorkspace(std::size_t nBins) +{ + RooRealVar x{"x", "x", 0, 1}; + x.setBins(nBins); + RooRealVar mu{"mu", "mu", 1.0, 0, 5}; + RooRealVar alpha{"alpha", "alpha", 0.3, -5, 5}; + TRandom3 rng{5}; + RooHistFactorySample::Spec spec; + spec.sampleName = "sample"; + RooHistFactorySample::HistoSys histoSys{"syst", &alpha, {}, {}}; + for (std::size_t i = 0; i < nBins; ++i) { + spec.contents.push_back(rng.Uniform(1, 2)); + histoSys.lo.push_back(0.9 * spec.contents.back()); + histoSys.hi.push_back(1.2 * spec.contents.back()); + } + spec.histoSys.push_back(histoSys); + spec.normFactors.push_back({"mu", &mu}); + RooHistFactorySample sample{"sample_shapes", "sample_shapes", RooArgList{x}, spec}; + RooHistFactoryPdf pdf{"model", "model", RooArgList{sample}}; + auto ws = std::make_unique("w"); + ws->import(pdf, RooFit::Silence()); + return ws; +} + +void expectSameSample(RooWorkspace &ws, RooWorkspace &wsRead) +{ + auto *sample = dynamic_cast(ws.function("sample_shapes")); + auto *sampleRead = dynamic_cast(wsRead.function("sample_shapes")); + ASSERT_NE(sampleRead, nullptr); + EXPECT_EQ(sampleRead->data().contents, sample->data().contents); + EXPECT_EQ(sampleRead->data().histoSysLo, sample->data().histoSysLo); + EXPECT_EQ(sampleRead->data().histoSysHi, sample->data().histoSysHi); + EXPECT_EQ(sampleRead->data().histoSysNames, sample->data().histoSysNames); + EXPECT_EQ(sampleRead->data().edges, sample->data().edges); + std::vector values(sample->numBins()); + std::vector valuesRead(sample->numBins()); + sample->binValues(values, false); + sampleRead->binValues(valuesRead, false); + EXPECT_EQ(valuesRead, values); +} + +} // namespace + +class RooHistFactoryPdfTest : public testing::TestWithParam {}; + +/// The new model structure gives the same values and likelihoods as the one before ROOT 6.44. +TEST_P(RooHistFactoryPdfTest, SameAsLegacyStructure) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + const int dim = GetParam(); + auto meas = makeMeasurement(dim); + auto wsOld = makeWorkspace(*meas, false); + auto wsNew = makeWorkspace(*meas, true); + + auto *pdfOld = wsOld->pdf("ch_model"); + auto *pdfNew = dynamic_cast(wsNew->pdf("ch_model")); + ASSERT_NE(pdfNew, nullptr); + EXPECT_EQ(pdfNew->numSamples(), 3u); + auto *mcOld = static_cast(wsOld->obj("ModelConfig")); + auto *mcNew = static_cast(wsNew->obj("ModelConfig")); + EXPECT_EQ(mcOld->GetNuisanceParameters()->size(), mcNew->GetNuisanceParameters()->size()); + EXPECT_EQ(mcOld->GetGlobalObservables()->size(), mcNew->GetGlobalObservables()->size()); + + RooArgSet obsOld; + RooArgSet obsNew; + pdfOld->getObservables(mcOld->GetObservables(), obsOld); + pdfNew->getObservables(mcNew->GetObservables(), obsNew); + + TRandom3 rng{42}; + for (int iPoint = 0; iPoint < 4; ++iPoint) { + if (iPoint > 0) { + setParameters(rng, *wsOld, *wsNew); + } + RooDataHist bins{"bins", "bins", obsOld}; + for (int i = 0; i < bins.numEntries(); ++i) { + obsOld.assign(*bins.get(i)); + obsNew.assign(*bins.get(i)); + // The same operations in the same order give the same values. + EXPECT_DOUBLE_EQ(pdfNew->getVal(), pdfOld->getVal()) << "bin " << i; + EXPECT_NEAR(pdfNew->getVal(obsNew), pdfOld->getVal(obsOld), 1e-12 * pdfOld->getVal(obsOld)) << "bin " << i; + } + EXPECT_NEAR(pdfNew->expectedEvents(&obsNew), pdfOld->expectedEvents(&obsOld), + 1e-12 * pdfOld->expectedEvents(&obsOld)); + + for (bool binned : {true, false}) { + pdfOld->setAttribute("BinnedLikelihood", binned); + pdfNew->setAttribute("BinnedLikelihood", binned); + using RooFit::GlobalObservables; + std::unique_ptr nllOld{ + mcOld->GetPdf()->createNLL(*wsOld->data("obsData"), GlobalObservables(*mcOld->GetGlobalObservables()))}; + std::unique_ptr nllNew{ + mcNew->GetPdf()->createNLL(*wsNew->data("obsData"), GlobalObservables(*mcNew->GetGlobalObservables()))}; + ASSERT_TRUE(std::isfinite(nllOld->getVal())) << "point " << iPoint << ", binned = " << binned; + EXPECT_NEAR(nllNew->getVal(), nllOld->getVal(), 1e-12 * std::abs(nllOld->getVal())) + << "point " << iPoint << ", binned = " << binned; + } + } +} + +#ifdef TEST_CODEGEN_AD +/// The likelihood with the code generation backend is the same as with the cpu backend. +TEST_P(RooHistFactoryPdfTest, Codegen) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto meas = makeMeasurement(GetParam()); + auto ws = makeWorkspace(*meas, true); + auto *mc = static_cast(ws->obj("ModelConfig")); + + TRandom3 rng{7}; + for (bool binned : {true, false}) { + ws->pdf("ch_model")->setAttribute("BinnedLikelihood", binned); + using RooFit::GlobalObservables; + std::unique_ptr nllCpu{mc->GetPdf()->createNLL( + *ws->data("obsData"), GlobalObservables(*mc->GetGlobalObservables()), RooFit::EvalBackend::Cpu())}; + std::unique_ptr nllCodegen{mc->GetPdf()->createNLL( + *ws->data("obsData"), GlobalObservables(*mc->GetGlobalObservables()), RooFit::EvalBackend::Codegen())}; + for (int iPoint = 0; iPoint < 3; ++iPoint) { + if (iPoint > 0) { + setParameters(rng, *ws, *ws); + } + ASSERT_TRUE(std::isfinite(nllCpu->getVal())) << "point " << iPoint << ", binned = " << binned; + EXPECT_NEAR(nllCodegen->getVal(), nllCpu->getVal(), 1e-10 * std::abs(nllCpu->getVal())) + << "point " << iPoint << ", binned = " << binned; + } + ws->loadSnapshot("NominalParamValues"); + } +} +#endif + +/// The samples and their data survive the writing to a file, with the data in the workspace object or in an RNTuple, +/// and the data is shared between clones. +TEST_P(RooHistFactoryPdfTest, PersistenceAndClones) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto meas = makeMeasurement(GetParam()); + auto ws = makeWorkspace(*meas, true); + + const std::string fileName = "testRooHistFactoryPdf_" + std::to_string(GetParam()) + ".root"; + { + TFile file(fileName.c_str(), "RECREATE"); + file.WriteObject(ws.get(), "ws"); + ws->Write("wsRNTuple"); + } + std::unique_ptr file{TFile::Open(fileName.c_str())}; + // Only RooWorkspace::Write() puts the data of the samples into an RNTuple. + TKey *key = file->GetKey("combined_histfactory_arrays"); + ASSERT_NE(key, nullptr); + EXPECT_STREQ(key->GetClassName(), "ROOT::RNTuple"); + EXPECT_EQ(file->GetKey("combined_histfactory_arrays_1"), nullptr); + + auto *pdf = dynamic_cast(ws->pdf("ch_model")); + for (const char *wsName : {"ws", "wsRNTuple"}) { + std::unique_ptr wsRead{file->Get(wsName)}; + ASSERT_NE(wsRead, nullptr) << wsName; + auto *pdfRead = dynamic_cast(wsRead->pdf("ch_model")); + ASSERT_NE(pdfRead, nullptr); + ASSERT_EQ(pdfRead->numSamples(), pdf->numSamples()); + + TRandom3 rng{3}; + setParameters(rng, *ws, *wsRead); + for (std::size_t i = 0; i < pdf->numSamples(); ++i) { + RooHistFactorySample const &sample = pdf->sample(i); + RooHistFactorySample const &sampleRead = pdfRead->sample(i); + EXPECT_EQ(sampleRead.sampleName(), sample.sampleName()); + EXPECT_EQ(sampleRead.data().contents, sample.data().contents); + EXPECT_EQ(sampleRead.data().histoSysHi, sample.data().histoSysHi); + EXPECT_EQ(sampleRead.data().edges, sample.data().edges); + std::vector values(sample.numBins()); + std::vector valuesRead(sample.numBins()); + sample.binValues(values, false); + sampleRead.binValues(valuesRead, false); + EXPECT_EQ(valuesRead, values) << wsName << " " << sample.sampleName(); + } + ws->loadSnapshot("NominalParamValues"); + } + + // A clone shares the data, until its interpolation codes are changed. + auto const &signal = *pdf->sample("signal"); + std::unique_ptr clone{static_cast(signal.clone())}; + EXPECT_EQ(&clone->data(), &signal.data()); + clone->setHistoSysInterpCode(*ws->var("alpha_shape1"), 0); + EXPECT_NE(&clone->data(), &signal.data()); + EXPECT_EQ(clone->data().histoSysCodes, (std::vector{0, 4})); + EXPECT_EQ(signal.data().histoSysCodes, (std::vector{4, 4})); +} + +/// With RooWorkspace::Write(), the sample data is not part of the workspace object in the file, so that the workspace +/// is not limited by the size limit of objects in ROOT files. Every time the workspace is written, it takes a new +/// RNTuple. +TEST(RooHistFactoryPdf, DataInRNTuple) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + const std::size_t nBins = 100000; + auto ws = makeLargeSampleWorkspace(nBins); + const std::string fileName = "testRooHistFactoryPdf_rntuple.root"; + { + TFile file(fileName.c_str(), "RECREATE"); + ws->Write(); + ws->Write("w2"); + } + TFile file(fileName.c_str()); + for (const char *name : {"w_histfactory_arrays", "w_histfactory_arrays_1"}) { + TKey *key = file.GetKey(name); + ASSERT_NE(key, nullptr) << name; + EXPECT_STREQ(key->GetClassName(), "ROOT::RNTuple"); + } + EXPECT_LT(file.GetKey("w")->GetObjlen(), nBins * sizeof(double)); + for (const char *name : {"w", "w2"}) { + std::unique_ptr wsRead{file.Get(name)}; + ASSERT_NE(wsRead, nullptr) << name; + expectSameSample(*ws, *wsRead); + } + + const std::string fileName2 = "testRooHistFactoryPdf_rntuple_2.root"; + ASSERT_TRUE(ws->writeToFile(fileName2.c_str())); + TFile file2(fileName2.c_str()); + ASSERT_NE(file2.GetKey("w_histfactory_arrays"), nullptr); + std::unique_ptr wsRead{file2.Get("w")}; + ASSERT_NE(wsRead, nullptr); + expectSameSample(*ws, *wsRead); +} + +/// Files in memory get the RNTuple too. +TEST(RooHistFactoryPdf, DataInRNTupleInMemoryFile) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto ws = makeLargeSampleWorkspace(1000); + TMemFile file("testRooHistFactoryPdf_memfile.root", "RECREATE"); + ws->Write(); + ASSERT_NE(file.GetKey("w_histfactory_arrays"), nullptr); + std::unique_ptr wsRead{file.Get("w")}; + ASSERT_NE(wsRead, nullptr); + expectSameSample(*ws, *wsRead); +} + +/// Without RooWorkspace::Write(), the data is written with the sample. +TEST(RooHistFactoryPdf, DataWithoutRNTuple) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto ws = makeLargeSampleWorkspace(1000); + + TBufferFile buffer{TBuffer::kWrite}; + buffer.WriteObject(ws.get()); + buffer.SetReadMode(); + buffer.SetBufferOffset(0); + std::unique_ptr wsRead{static_cast(buffer.ReadObject(RooWorkspace::Class()))}; + ASSERT_NE(wsRead, nullptr); + expectSameSample(*ws, *wsRead); + + TMemFile file("testRooHistFactoryPdf_inline.root", "RECREATE"); + file.WriteObject(ws.get(), "w"); + file.WriteObject(ws->function("sample_shapes"), "sample"); + for (TObject *key : *file.GetListOfKeys()) { + EXPECT_STRNE(static_cast(key)->GetClassName(), "ROOT::RNTuple"); + } + std::unique_ptr wsFromFile{file.Get("w")}; + ASSERT_NE(wsFromFile, nullptr); + expectSameSample(*ws, *wsFromFile); +} + +/// Integrals over ranges that don't align with the bins take the fraction of the bins in the range. +TEST(RooHistFactoryPdf, RangedIntegral) +{ + RooRealVar x{"x", "x", 0, 4}; + x.setBins(4); + RooRealVar mu{"mu", "mu", 2.0}; + RooHistFactorySample::Spec spec; + spec.sampleName = "sample"; + spec.contents = {1., 2., 3., 4.}; + spec.normFactors.push_back({"mu", &mu}); + RooHistFactorySample sample{"sample", "sample", RooArgList{x}, spec}; + RooHistFactoryPdf pdf{"pdf", "pdf", RooArgList{sample}}; + + x.setRange("r", 0.5, 2.25); + std::unique_ptr integral{sample.createIntegral(x, "r")}; + EXPECT_DOUBLE_EQ(integral->getVal(), mu.getVal() * (0.5 * 1. + 2. + 0.25 * 3.)); + std::unique_ptr full{sample.createIntegral(x)}; + EXPECT_DOUBLE_EQ(full->getVal(), mu.getVal() * 10.); + RooArgSet xSet{x}; + EXPECT_DOUBLE_EQ(pdf.expectedEvents(&xSet), mu.getVal() * 10.); + x.setVal(2.5); + RooArgSet normSet{x}; + EXPECT_DOUBLE_EQ(pdf.getVal(normSet), 3. / 10.); +} + +INSTANTIATE_TEST_SUITE_P(RooHistFactoryPdf, RooHistFactoryPdfTest, testing::Values(1, 2), + [](testing::TestParamInfo const ¶mInfo) { + return std::to_string(paramInfo.param) + "D"; + }); diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index aac0e3329f720..17c518ac014b6 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -96,6 +96,10 @@ class RooJSONFactoryWSTool { /// their data set of one entry in the combined data set, so that the RooSimultaneous is the sampling /// distribution of all measurements. This option and the form of the resulting workspace may still change. bool importConstraintsAsChannels = false; + /// Import each `histfactory_dist` as a RooHistFactoryPdf of RooHistFactorySample objects. If false, or if a + /// channel has custom modifiers that depend on the observables, the channel is imported as before ROOT 6.44, + /// with several RooFit objects for each histogram. + bool useHistFactoryPdf = true; }; static Config &config(); diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 5a7a711bb6e18..139c69b660f2c 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -393,13 +395,10 @@ void configureGammasWithConstraints(RooArgList const &gammas, std::vector const &parnames, std::vector const &vals, - HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, - double gammaMax, double minSigma, std::size_t nBins, std::string const &context, - bool configureGammas = true) +// The gamma parameters of a modifier that scales each bin, in the order of the bins (last axis fastest). +RooArgList createGammas(RooJSONFactoryWSTool &tool, std::string const &sysname, + std::vector const &parnames, RooArgSet const &observables, double gammaMax, + std::size_t nBins) { RooWorkspace &ws = *tool.workspace(); @@ -411,9 +410,28 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, gammas.add(existing ? *existing : getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); } + // Documents before HS3 0.3 didn't define the order of the parameters of multi-dimensional modifiers. + checkParamHistFuncOrderIsDefined(tool.hs3Version(), observables, "the modifier '" + sysname + "'"); + return gammas; +} + +void constrainGammas(RooJSONFactoryWSTool &tool, std::string const &sysname, RooArgList const &gammas, + std::vector const &vals, HS3::GammaConstraint const &constraint, RooArgSet &constraints, + double minSigma, std::string const &context, bool configureGammas); + +// Create the ParamHistFunc of the gamma parameters, together with their constraints, see constrainGammas(). +ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, std::string const &sysname, + std::vector const &parnames, std::vector const &vals, + HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, + double gammaMax, double minSigma, std::size_t nBins, std::string const &context, + bool configureGammas = true) +{ + RooWorkspace &ws = *tool.workspace(); + + RooArgList gammas = createGammas(tool, sysname, parnames, observables, gammaMax, nBins); + // The gammas and `vals` are given in the order of HS3 (last axis fastest), which is also the order in which they // are paired below. ParamHistFunc expects its parameters in TH1 order. - checkParamHistFuncOrderIsDefined(tool.hs3Version(), observables, "the modifier '" + sysname + "'"); RooArgList gammasInTH1Order = toParamHistFuncOrder(gammas, observables); // Shape modifiers of the same name in a channel share the function, so they need the same parameters. @@ -427,6 +445,17 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, } auto &phf = tool.wsEmplace(phfname, observables, gammasInTH1Order); + constrainGammas(tool, sysname, gammas, vals, constraint, constraints, minSigma, context, configureGammas); + + return phf; +} + +// Create the constraints of the gamma parameters. With a constraint type, RooFit creates them from the relative +// uncertainties `vals`, like HistFactory. +void constrainGammas(RooJSONFactoryWSTool &tool, std::string const &sysname, RooArgList const &gammas, + std::vector const &vals, HS3::GammaConstraint const &constraint, RooArgSet &constraints, + double minSigma, std::string const &context, bool configureGammas) +{ auto addGammaConstraints = [&](Constraint::Type type) { for (auto const &term : createGammaConstraints(gammas, vals, minSigma, type).constraints) { constraints.add(tool.wsImport(*term)); @@ -451,7 +480,7 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, } if (vals.empty()) { - return phf; + return; } std::visit(HS3::overloaded{ @@ -478,8 +507,6 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, }, }, constraint); - - return phf; } // The relative uncertainty that the constraint distribution of a gamma encodes, see gammaRelUncertainty(). Constraints @@ -659,6 +686,134 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample) } } +// Collects the bin contents and modifiers of a sample for a RooHistFactorySample, creating the parameters and their +// constraints like buildSample(). +RooHistFactorySample::Spec buildSampleSpec(ChannelContext const &ctx, HS3::Sample const &sample, + RooArgList const &statGammas, std::map &shapeGammas) +{ + using BinFactorType = RooHistFactorySample::BinFactorType; + + RooJSONFactoryWSTool &tool = ctx.tool; + std::string const &sampleName = sample.name; + + RooHistFactorySample::Spec spec; + spec.sampleName = sampleName; + spec.contents = sample.contents; + if (sample.errors) { + spec.errors = *sample.errors; + } + + // The validation made sure that the channel has a "default_interpolation" if a modifier doesn't give its own. + auto interpolationCode = [&](std::optional const &interpolation, + InterpolationClass interpolationClass, const char *type, std::string const &name) { + return codeFromInterpolation(interpolation ? *interpolation : ctx.dist.defaultInterpolation.value(), + interpolationClass, + std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + + sampleName + "' of channel '" + ctx.channelName + "'"); + }; + + // Like in buildSample(), the parameters of the staterror come first. + for (HS3::Modifier const &modifier : sample.modifiers) { + if (auto const *statError = std::get_if(&modifier)) { + spec.binFactors.push_back({statError->name, BinFactorType::StatError, statGammas}); + break; + } + } + + for (HS3::Modifier const &modifier : sample.modifiers) { + std::visit( + HS3::overloaded{ + [&](HS3::StatError const &) { + // see above + }, + [&](HS3::NormFactor const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 1., -3, 5, mod.name); + // Like in buildSample(), an error that is already set is kept. + if (auto *var = dynamic_cast(¶m); var && !var->hasError()) { + var->setError(0.0); + } + spec.normFactors.push_back({mod.name, ¶m}); + // for norm factors, constraints are optional + addConstraint(tool, mod.constraint, param, 1.0, "normfactor", mod.name, sampleName, ctx.constraints); + }, + [&](HS3::NormSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + const int interp = + interpolationCode(mod.interpolation, InterpolationClass::Flexible, "normsys", mod.name); + double low = mod.lo; + double high = mod.hi; + // The same cutoff of variations below zero as in buildSample(). + if (interp == 4 && low <= 0) + low = std::numeric_limits::epsilon(); + if (interp == 4 && high <= 0) + high = std::numeric_limits::epsilon(); + spec.normSys.push_back({mod.name, ¶m, low, high, interp}); + addSysConstraint(tool, mod.constraint, param, "normsys", mod.name, sampleName, ctx.constraints); + }, + [&](HS3::HistoSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + spec.histoSys.push_back( + {mod.name, ¶m, mod.lo.contents, mod.hi.contents, + interpolationCode(mod.interpolation, InterpolationClass::Piecewise, "histosys", mod.name)}); + addSysConstraint(tool, mod.constraint, param, "histosys", mod.name, sampleName, ctx.constraints); + }, + [&](HS3::ShapeSys const &mod) { + const bool constrained = std::visit( + HS3::overloaded{[](HS3::Unconstrained) { return false; }, [](HS3::ConstantGammas) { return false; }, + [&](HS3::ConstraintType) { return !mod.vals.empty(); }, + [](HS3::GammaConstraintPdfs const &constraints) { return anyNamed(constraints); }}, + mod.constraint); + RooArgList gammas = createGammas(tool, mod.name, mod.parameters, ctx.observables, + defaultShapeSysGammaMax, sample.contents.size()); + // Shape modifiers of the same name in a channel share their parameters. + auto found = shapeGammas.find(mod.name); + if (found == shapeGammas.end()) { + shapeGammas.emplace(mod.name, gammas); + } else { + bool same = found->second.size() == gammas.size(); + for (std::size_t i = 0; same && i < gammas.size(); ++i) { + same = &found->second[i] == &gammas[i]; + } + if (!same) { + RooJSONFactoryWSTool::error("the modifier '" + mod.name + "' has other parameters than another " + + "modifier of this name in the channel"); + } + } + constrainGammas(tool, mod.name, gammas, mod.vals, mod.constraint, ctx.constraints, minShapeUncertainty, + "the shapesys modifier '" + mod.name + "' of sample '" + sampleName + "'", + /*configureGammas=*/true); + spec.binFactors.push_back( + {mod.name, constrained ? BinFactorType::ShapeSys : BinFactorType::ShapeFactor, gammas}); + }, + [&](HS3::CustomModifier const &mod) { + // The function can come later in the document. + RooAbsReal *obj = tool.request(mod.name, ctx.channelName); + spec.normFactors.push_back({mod.name, obj}); + }, + }, + modifier); + } + + return spec; +} + +// RooHistFactorySample doesn't support custom modifiers that depend on the observables. Channels with such modifiers +// are imported with the structure before ROOT 6.44. +bool hasObservableDependentModifiers(RooJSONFactoryWSTool &tool, HS3::HistFactoryDist const &dist, + RooArgSet const &observables, std::string const &channelName) +{ + for (auto const &sample : dist.samples) { + for (auto const &modifier : sample.modifiers) { + if (auto const *custom = std::get_if(&modifier)) { + if (tool.request(custom->name, channelName)->dependsOn(observables)) { + return true; + } + } + } + } + return false; +} + // The constraint of the gamma parameters of the staterror modifiers. Without one, the gamma parameters are free. HS3::GammaConstraint statErrorConstraint(HS3::HistFactoryDist const &dist) { @@ -1359,13 +1514,177 @@ bool isValidChannel(ExportedChannel const &channel, std::string const &pdfname) return !HS3::hasErrors(diagnostics); } +// Reads a RooHistFactoryPdf, which has the modifiers of its samples explicitly. +ExportedChannel readHistFactoryPdf(RooJSONFactoryWSTool *tool, const std::string &pdfname, RooHistFactoryPdf const &pdf, + std::vector const &channelConstraints) +{ + using BinFactorType = RooHistFactorySample::BinFactorType; + + ExportedChannel channel; + RooWorkspace *ws = tool->workspace(); + + channel.name = pdfname; + erasePrefix(channel.name, "model_"); + eraseSuffix(channel.name, "_model"); + channel.observables = std::make_unique(pdf.observables()); + + // Like in readChannel(), the modifiers name only the constraint terms of the channel. + auto constraintTerm = [&](RooAbsPdf const *pdf) -> std::optional { + if (std::find(channelConstraints.begin(), channelConstraints.end(), pdf) == channelConstraints.end()) { + return std::nullopt; + } + channel.constraints.emplace(pdf->GetName(), pdf); + return HS3::DistributionRef{pdf->GetName()}; + }; + auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { + auto ref = constraintTerm(pdf); + return ref ? HS3::Constraint{HS3::ConstraintPdf{*ref}} : HS3::Unconstrained{}; + }; + // The constraints of the gammas, in the order of the bins. If none is named, the gammas are free. + auto gammaConstraints = [&](std::vector const &gammas, bool alsoByName) -> HS3::GammaConstraint { + HS3::GammaConstraintPdfs pdfs; + for (RooAbsReal *g : gammas) { + RooAbsPdf *constraintPdf = findConstraint(g, channelConstraints); + if (!constraintPdf && alsoByName) { + constraintPdf = ws->pdf(constraintName(g->GetName())); + } + pdfs.pdfs.push_back(constraintTerm(constraintPdf)); + } + if (anyNamed(pdfs)) { + return pdfs; + } + return HS3::Unconstrained{}; + }; + + for (std::size_t iSample = 0; iSample < pdf.numSamples(); ++iSample) { + RooHistFactorySample const &hfSample = pdf.sample(iSample); + RooHistFactorySample::Data const &data = hfSample.data(); + auto param = [&](int index) { return &static_cast(hfSample.parameters()[index]); }; + const std::size_t nBins = data.numBins(); + + HS3::Sample sample; + sample.name = data.sampleName; + sample.contents = data.contents; + if (!data.errors.empty()) { + sample.errors = data.errors; + } + + // The modifiers by type, which are written in this order, like for the models before ROOT 6.44. + std::vector normFactors; + std::vector normSys; + std::vector histoSys; + std::vector shapeSys; + std::vector custom; + std::optional statError; + + for (std::size_t i = 0; i < data.normFactorParams.size(); ++i) { + RooAbsReal *par = param(data.normFactorParams[i]); + if (auto constVar = dynamic_cast(par)) { + if (constVar->getVal() != 1.) { + normFactors.push_back({constVar->GetName(), {constVar->GetName()}, HS3::Unconstrained{}}); + } + } else if (dynamic_cast(par)) { + normFactors.push_back( + {par->GetName(), {par->GetName()}, constraint(findConstraint(par, channelConstraints))}); + } else { + custom.push_back({par->GetName()}); + channel.customModifiers.push_back(par); + } + } + + for (std::size_t i = 0; i < data.normSysParams.size(); ++i) { + RooAbsReal *par = param(data.normSysParams[i]); + RooAbsPdf const *constraintPdf = findConstraint(par, channelConstraints); + checkModifierParameter(*par, constraintPdf); + const std::string context = "normsys modifier '" + data.normSysNames[i] + "' in sample '" + sample.name + + "' of channel '" + channel.name + "'"; + normSys.push_back({data.normSysNames[i], + {par->GetName()}, + data.normSysHi[i], + data.normSysLo[i], + constraint(constraintPdf), + interpolationFromCode(data.normSysCodes[i], InterpolationClass::Flexible, context)}); + } + + for (std::size_t i = 0; i < data.histoSysParams.size(); ++i) { + RooAbsReal *par = param(data.histoSysParams[i]); + RooAbsPdf const *constraintPdf = findConstraint(par, channelConstraints); + checkModifierParameter(*par, constraintPdf); + const std::string context = "histosys modifier '" + data.histoSysNames[i] + "' in sample '" + sample.name + + "' of channel '" + channel.name + "'"; + auto variation = [&](std::vector const &all) { + return HS3::HistoSysVariation{{all.begin() + i * nBins, all.begin() + (i + 1) * nBins}, std::nullopt}; + }; + histoSys.push_back({data.histoSysNames[i], + {par->GetName()}, + variation(data.histoSysHi), + variation(data.histoSysLo), + constraint(constraintPdf), + interpolationFromCode(data.histoSysCodes[i], InterpolationClass::Piecewise, context)}); + } + + for (std::size_t i = 0; i < data.binFactorNames.size(); ++i) { + std::vector gammas; + for (std::size_t iBin = 0; iBin < nBins; ++iBin) { + gammas.push_back(param(data.binFactorParams[i * nBins + iBin])); + } + if (static_cast(data.binFactorTypes[i]) == BinFactorType::StatError) { + statError.emplace(); + statError->name = data.binFactorNames[i]; + for (std::size_t iBin = 0; iBin < nBins; ++iBin) { + RooAbsReal *g = gammas[iBin]; + statError->parameters.push_back({g->GetName()}); + // To reconstruct the statistical uncertainties if the sample doesn't have them. + channel.totYield[iBin] += sample.contents[iBin]; + channel.totYield2[iBin] += sample.contents[iBin] * sample.contents[iBin]; + if (RooAbsPdf *constraintPdf = findConstraint(g, channelConstraints)) { + channel.relErrors[iBin] = constraintRelError(*constraintPdf, *g); + } + } + statError->constraint = gammaConstraints(gammas, false); + } else { + HS3::ShapeSys &sys = shapeSys.emplace_back(); + sys.name = data.binFactorNames[i]; + for (RooAbsReal *g : gammas) { + sys.parameters.push_back({g->GetName()}); + } + // Both are written as "shapefactor": the constrained parameters name their constraint distribution in + // "constraints", the others are free. + sys.constraint = gammaConstraints(gammas, true); + } + } + + sortByName(normFactors); + sortByName(normSys); + sortByName(histoSys); + sortByName(shapeSys); + auto append = [&](auto &modifiers) { + sample.modifiers.insert(sample.modifiers.end(), std::make_move_iterator(modifiers.begin()), + std::make_move_iterator(modifiers.end())); + }; + append(normFactors); + append(normSys); + append(histoSys); + append(shapeSys); + append(custom); + if (statError) { + sample.modifiers.push_back(std::move(*statError)); + } + + channel.dist.samples.push_back(std::move(sample)); + } + + sortByName(channel.dist.samples); + return channel; +} + // The statistical uncertainties of the samples are reconstructed from the constraints of the gamma parameters, with // the (arbitrary) convention that all samples have the same relative uncertainty. void configureStatError(ExportedChannel &channel) { for (HS3::Sample &sample : channel.dist.samples) { - if (std::none_of(sample.modifiers.begin(), sample.modifiers.end(), - [](auto const &m) { return std::holds_alternative(m); })) { + if (sample.errors || std::none_of(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); })) { continue; } std::vector errors(sample.contents.size()); @@ -1522,20 +1841,24 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname return false; } - for (RooAbsArg *sample : sumpdf->funcList()) { - if (!dynamic_cast(sample) && !dynamic_cast(sample)) { - return false; + ExportedChannel channel; + if (auto *hfPdf = dynamic_cast(sumpdf)) { + channel = readHistFactoryPdf(tool, pdfname, *hfPdf, constraints); + } else { + for (RooAbsArg *sample : sumpdf->funcList()) { + if (!dynamic_cast(sample) && !dynamic_cast(sample)) { + return false; + } } - } - - ExportedChannel channel = readChannel(tool, pdfname, sumpdf, constraints); - // Writing a formula as several modifiers must not be what makes the channel invalid, e.g. because a parameter that - // occurs twice in the formula becomes two normfactors of the same name. If it is, the formulas are written as - // custom modifiers instead. - if (channel.decomposedFormulas && !isValidChannel(channel, pdfname)) { - ExportedChannel plain = readChannel(tool, pdfname, sumpdf, constraints, /*decomposeFormulas=*/false); - if (isValidChannel(plain, pdfname)) { - channel = std::move(plain); + channel = readChannel(tool, pdfname, sumpdf, constraints); + // Writing a formula as several modifiers must not be what makes the channel invalid, e.g. because a parameter + // that occurs twice in the formula becomes two normfactors of the same name. If it is, the formulas are written + // as custom modifiers instead. + if (channel.decomposedFormulas && !isValidChannel(channel, pdfname)) { + ExportedChannel plain = readChannel(tool, pdfname, sumpdf, constraints, /*decomposeFormulas=*/false); + if (isValidChannel(plain, pdfname)) { + channel = std::move(plain); + } } } HS3::HistFactoryDist &dist = channel.dist; @@ -1624,6 +1947,7 @@ const bool registered = []() { RooFit::JSONIO::registerImporter("histfactory_dist", true); RooFit::JSONIO::registerExporter(RooProdPdf::Class(), true); RooFit::JSONIO::registerExporter(RooRealSumPdf::Class(), true); + RooFit::JSONIO::registerExporter(RooHistFactoryPdf::Class(), true); return true; }(); @@ -1633,6 +1957,12 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c { RooArgSet observables = createObservables(dist.axes); + std::string channelName = name; + erasePrefix(channelName, "model_"); + + const bool useHistFactoryPdf = RooJSONFactoryWSTool::config().useHistFactoryPdf && + !hasObservableDependentModifiers(tool, dist, observables, channelName); + // The yields and their variances by bin, summed over the samples with a staterror modifier. std::vector sumW; std::vector sumW2; @@ -1659,10 +1989,8 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c } } - std::string channelName = name; - erasePrefix(channelName, "model_"); - RooAbsArg *mcStatObject = nullptr; + RooArgList statGammas; RooArgSet constraints; if (!sumW.empty()) { // The relative uncertainties. The gammas of bins without a positive yield are set constant later. @@ -1675,19 +2003,39 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c // range, the constness and the step width of the gammas come from the document, and nothing is derived from // the errors of the samples. Older documents relied on RooFit configuring the gammas from them. const bool configureGammas = tool.hs3Version() < HS3::Version{0, 3}; - mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, - statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, - dist.statError.relThreshold, errs.size(), - "the staterror modifier of channel '" + name + "'", configureGammas); + const std::string context = "the staterror modifier of channel '" + name + "'"; + if (useHistFactoryPdf) { + statGammas.add(createGammas(tool, "stat_" + channelName, gammaParnames, observables, defaultStatErrorGammaMax, + errs.size())); + constrainGammas(tool, "stat_" + channelName, statGammas, errs, statErrorConstraint(dist), constraints, + dist.statError.relThreshold, context, configureGammas); + } else { + mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, + statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, + dist.statError.relThreshold, errs.size(), context, configureGammas); + } } ChannelContext ctx{tool, dist, observables, name, channelName, mcStatObject, constraints}; RooArgList funcs; RooArgList coefs; + std::map shapeGammas; for (HS3::Sample const &sample : dist.samples) { + const std::string shapesName = name + "_" + sample.name + "_shapes"; + if (useHistFactoryPdf) { + if (tool.workspace()->arg(shapesName.c_str())) { + RooJSONFactoryWSTool::error("the object '" + shapesName + "' for sample '" + sample.name + + "' of channel '" + channelName + "' has the name of an existing object"); + } + RooArgList obsList{observables}; + funcs.add(tool.wsEmplace(shapesName, obsList, + buildSampleSpec(ctx, sample, statGammas, shapeGammas))); + funcs[funcs.size() - 1].SetTitle(sample.name.c_str()); + continue; + } buildSample(ctx, sample); - funcs.add(*tool.request(name + "_" + sample.name + "_shapes", name)); + funcs.add(*tool.request(shapesName, name)); coefs.add(*tool.request(name + "_" + sample.name + "_scaleFactors", name)); } @@ -1710,7 +2058,9 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c } // With constraints, the channel is the product of the sum of the samples and the constraints. - auto &sum = tool.wsEmplace(constraints.empty() ? name : channelName + "_model", funcs, coefs, true); + const std::string sumName = constraints.empty() ? name : channelName + "_model"; + auto &sum = useHistFactoryPdf ? tool.wsEmplace(sumName, funcs) + : tool.wsEmplace(sumName, funcs, coefs, true); sum.SetTitle(name.c_str()); // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation diff --git a/roofit/hs3/test/HS3TestUtils.h b/roofit/hs3/test/HS3TestUtils.h index cfa9b2d35e35d..6f46905b7a4ed 100644 --- a/roofit/hs3/test/HS3TestUtils.h +++ b/roofit/hs3/test/HS3TestUtils.h @@ -121,6 +121,10 @@ inline ConfigFlag importConstraintsAsChannels(bool value) { return {&RooJSONFactoryWSTool::Config::importConstraintsAsChannels, value}; } +inline ConfigFlag useHistFactoryPdf(bool value) +{ + return {&RooJSONFactoryWSTool::Config::useHistFactoryPdf, value}; +} /// Sets flags of RooJSONFactoryWSTool::config() for its lifetime. class ScopedConfig { diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 38e180e352700..8205eeae5a67d 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include #include #include @@ -91,6 +93,53 @@ std::string jsonString(RooFit::Detail::JSONTree &tree) return stream.str(); } +// The sample "sig" of the channel of HistFactoryDoc. +RooHistFactorySample *sigSample(RooWorkspace &ws) +{ + return object(ws, "model_channel0_sig_shapes"); +} + +// The interpolation codes of the NormSys or HistoSys modifiers of a sample, by parameter name. +std::map interpolationCodes(RooHistFactorySample const &sample, bool normSys) +{ + auto const &data = sample.data(); + auto const ¶ms = normSys ? data.normSysParams : data.histoSysParams; + auto const &codes = normSys ? data.normSysCodes : data.histoSysCodes; + std::map out; + for (std::size_t i = 0; i < params.size(); ++i) { + out[sample.parameters()[params[i]].GetName()] = codes[i]; + } + return out; +} + +// The type of the modifier of a sample that scales each bin with its own parameter, by name. +std::optional +binFactorType(RooHistFactorySample const &sample, std::string const &name) +{ + auto const &data = sample.data(); + for (std::size_t i = 0; i < data.binFactorNames.size(); ++i) { + if (data.binFactorNames[i] == name) { + return static_cast(data.binFactorTypes[i]); + } + } + return std::nullopt; +} + +// The parameters of the modifier of a sample that scales each bin with its own parameter, by name. +RooArgList binFactorParams(RooHistFactorySample const &sample, std::string const &name) +{ + auto const &data = sample.data(); + RooArgList params; + for (std::size_t i = 0; i < data.binFactorNames.size(); ++i) { + if (data.binFactorNames[i] == name) { + for (std::size_t iBin = 0; iBin < data.numBins(); ++iBin) { + params.add(sample.parameters()[data.binFactorParams[i * data.numBins() + iBin]]); + } + } + } + return params; +} + // Builds a single-bin-observable RooDataHist filled with the given bin contents. Returns an owning pointer so that it // can be handed to the RooHistFunc constructor that takes ownership (the const-reference overload keeps only an unowned // pointer, which would dangle for a temporary). @@ -1551,13 +1600,10 @@ TEST(RooFitHS3, HistFactoryInterpolationCodeMapping) EXPECT_FALSE(exported.has("distributions/model_channel0/samples/sig/modifiers/modifier/interpolation")); Imported roundTripped{exported}; - if (isNormSys) { - auto *interpolation = object(roundTripped, "sig_channel0_epsilon"); - EXPECT_EQ(interpolation->interpolationCodes(), std::vector{testCase.canonicalCode}); - } else { - auto *interpolation = object(roundTripped, "histoSys_model_channel0_sig"); - EXPECT_EQ(interpolation->interpolationCodes(), std::vector{testCase.canonicalCode}); - } + // The codes of NormSys are the ones of FlexibleInterpVar, and the codes of HistoSys the ones of + // PiecewiseInterpolation. + const std::map codes = interpolationCodes(*sigSample(roundTripped), isNormSys); + EXPECT_EQ(codes, (std::map{{"alpha_modifier", testCase.canonicalCode}})); } } @@ -1602,18 +1648,8 @@ TEST(RooFitHS3, HistFactoryInterpolationDefaultsAndOverrides) {"name": "shape_quadratic", "interpolation": {"type": "add", "in": "poly2", "out": "poly1"}}]}]})")); Imported roundTripped{exported}; - auto *shapeInterpolation = object(roundTripped, "histoSys_model_channel0_sig"); - ASSERT_EQ(shapeInterpolation->paramList().size(), 2u); - for (std::size_t i = 0; i < shapeInterpolation->paramList().size(); ++i) { - const std::string parameter = shapeInterpolation->paramList().at(i)->GetName(); - if (parameter == "alpha_shape_default") { - EXPECT_EQ(shapeInterpolation->interpolationCodes()[i], 4); - } else if (parameter == "alpha_shape_quadratic") { - EXPECT_EQ(shapeInterpolation->interpolationCodes()[i], 2); - } else { - FAIL() << "Unexpected histosys parameter " << parameter; - } - } + EXPECT_EQ(interpolationCodes(*sigSample(roundTripped), false), + (std::map{{"alpha_shape_default", 4}, {"alpha_shape_quadratic", 2}})); const std::string implicitModifiers = R"( {"name":"norm","parameter":"alpha_norm","type":"normsys","data":{"lo":0.8,"hi":1.2}}, @@ -1669,8 +1705,7 @@ std::vector structuredInterpolationYields(RooWorkspace &ws) ws.var("alpha_shape")->setVal(theta); for (int bin = 0; bin < 2; ++bin) { ws.var("x")->setBin(bin); - yields.push_back(ws.function("model_channel0_sig_shapes")->getVal(observables) * - ws.function("model_channel0_sig_scaleFactors")->getVal()); + yields.push_back(ws.function("model_channel0_sig_shapes")->getVal(observables)); } } return yields; @@ -1690,12 +1725,8 @@ INSTANTIATE_TEST_SUITE_P(HistFactoryStructuredInterpolation, RoundTrip, TEST(RooFitHS3, HistFactoryStructuredInterpolationRoundTrip) { Imported ws{structuredInterpolationJSON()}; - auto *norm = dynamic_cast(ws.function("sig_channel0_epsilon")); - auto *shape = dynamic_cast(ws.function("histoSys_model_channel0_sig")); - ASSERT_NE(norm, nullptr); - ASSERT_NE(shape, nullptr); - EXPECT_EQ(norm->interpolationCodes()[0], 4); - EXPECT_EQ(shape->interpolationCodes()[0], 2); + EXPECT_EQ(interpolationCodes(*sigSample(ws), true).at("alpha_norm"), 4); + EXPECT_EQ(interpolationCodes(*sigSample(ws), false).at("alpha_shape"), 2); } TEST(RooFitHS3, HistFactoryStructuredInterpolationValidation) @@ -1759,8 +1790,12 @@ TEST(RooFitHS3, HistFactoryZeroYieldBin) } } +// The RooFit objects of the HistFactory models before ROOT 6.44 don't store the names of the modifiers. The +// exporter names them after their parameters, and combines the modifiers that get the same name. TEST(RooFitHS3, HistFactoryDuplicateModifiersAreCombined) { + ScopedConfig config{{useHistFactoryPdf(false)}}; + // The two "poly" modifiers deliberately use implicit code 4 and explicit // code 5, which are semantic aliases in FlexibleInterpVar. Imported ws1{HistFactoryDoc{"0.1.90", R"( @@ -1849,9 +1884,55 @@ TEST(RooFitHS3, HistFactoryDuplicateModifiersAreCombined) EXPECT_TRUE(sentinelErrors.empty()) << sentinelErrors; } -// Duplicate modifiers that cannot be combined into one are rejected on export. +// A RooHistFactoryPdf keeps the names of the modifiers. Modifiers with the same name and type are combined in the +// export, and the others are kept as they are, even if they have the same parameter. +TEST(RooFitHS3, HistFactoryDuplicateModifiersRoundTrip) +{ + Imported ws1{HistFactoryDoc{"0.1.90", R"( + {"name": "norm", "parameter": "alpha_norm", "type": "normsys", "interpolation": 1, "data": {"lo": 0.8, "hi": 1.2}}, + {"name": "norm", "parameter": "alpha_norm", "type": "normsys", "interpolation": 1, "data": {"lo": 0.9, "hi": 1.1}}, + {"name": "shape_input_1", "parameter": "alpha_shape", "type": "histosys", + "data": {"lo": {"contents": [8.0, 18.0]}, "hi": {"contents": [12.0, 22.0]}}}, + {"name": "shape_input_2", "parameter": "alpha_shape", "type": "histosys", + "data": {"lo": {"contents": [9.0, 16.0]}, "hi": {"contents": [11.0, 24.0]}}})"}}; + + Doc exported{ws1}; + EXPECT_THAT(exported.warnings(), HasSubstr("combined 2 duplicate modifiers named 'norm' of type 'normsys'")); + EXPECT_THAT(exported.warnings(), Not(HasSubstr("of type 'histosys'"))); + EXPECT_EQ(exported.names("distributions/model_channel0/samples/sig/modifiers"), + (std::vector{"norm", "shape_input_1", "shape_input_2"})); + + Imported ws2{exported}; + RooHistFactorySample *sample = sigSample(ws2); + ASSERT_EQ(sample->data().normSysNames.size(), 1u); + EXPECT_DOUBLE_EQ(sample->data().normSysLo[0], 0.8 * 0.9); + EXPECT_DOUBLE_EQ(sample->data().normSysHi[0], 1.2 * 1.1); + EXPECT_EQ(sample->data().histoSysNames, (std::vector{"shape_input_1", "shape_input_2"})); + + for (double norm : {-0.6, 0.8, 1.3}) { + for (double shape : {-0.4, 0.35, 1.2}) { + for (int bin = 0; bin < 2; ++bin) { + auto prediction = [&](RooWorkspace &ws) { + ws.var("x")->setBin(bin); + ws.var("alpha_norm")->setVal(norm); + ws.var("alpha_shape")->setVal(shape); + return ws.function("model_channel0_sig_shapes")->getVal(); + }; + const double before = prediction(ws1); + EXPECT_NEAR(prediction(ws2), before, std::abs(before) * 1e-13) + << "bin=" << bin << ", norm=" << norm << ", shape=" << shape; + } + } + } +} + +// Duplicate modifiers that cannot be combined into one are rejected on export. The duplicates come from naming the +// modifiers after their parameters, which the exporter does for the RooFit objects of the HistFactory models before +// ROOT 6.44. A RooHistFactoryPdf keeps the names of the modifiers. TEST(RooFitHS3, HistFactoryIncompatibleDuplicateModifiers) { + ScopedConfig config{{useHistFactoryPdf(false)}}; + // Two normsys modifiers, "input_1" and "input_2", with the given further keys. auto normsys = [](std::string const &keys1, std::string const &keys2) { return R"({"name": "input_1", "type": "normsys", "data": {"lo": 0.8, "hi": 1.2}, )" + keys1 + R"(}, @@ -1929,12 +2010,13 @@ TEST(RooFitHS3, HistFactoryHistoSysOfTheSameNameInOlderDocuments) "there is another histosys modifier named 'shape' in the sample"); Imported ws{HistFactoryDoc{"0.2", modifiers}}; - auto *interp = object(ws, "histoSys_model_channel0_sig"); - ASSERT_EQ(interp->paramList().size(), 2u); - EXPECT_EQ(interp->paramList().at(0), interp->paramList().at(1)); - auto high = [&](int i, int bin) { - return static_cast(interp->highList().at(i))->dataHist().weight(bin); - }; + RooHistFactorySample *sample = sigSample(ws); + RooHistFactorySample::Data const &data = sample->data(); + EXPECT_EQ(data.histoSysNames, (std::vector{"shape", "shape"})); + ASSERT_EQ(data.histoSysParams.size(), 2u); + EXPECT_EQ(data.histoSysParams[0], data.histoSysParams[1]); + // The variations are stored one after the other. + auto high = [&](int i, int bin) { return data.histoSysHi[i * data.numBins() + bin]; }; EXPECT_DOUBLE_EQ(high(0, 1), 22.0); EXPECT_DOUBLE_EQ(high(1, 1), 24.0); @@ -1942,15 +2024,14 @@ TEST(RooFitHS3, HistFactoryHistoSysOfTheSameNameInOlderDocuments) Doc exported{ws}; EXPECT_EQ(exported.names("distributions/model_channel0/samples/sig/modifiers"), std::vector{"shape"}); Imported ws2{exported}; - auto *interp2 = object(ws2, "histoSys_model_channel0_sig"); - ASSERT_EQ(interp2->paramList().size(), 1u); - // Both functions give the same for a shifted parameter. (expectRoundTrip() cannot make this comparison: the - // variations of the first modifier and of the combined one have the same names.) + RooHistFactorySample *sample2 = sigSample(ws2); + EXPECT_EQ(sample2->data().histoSysNames, std::vector{"shape"}); + // Both samples give the same for a shifted parameter. ws.var("alpha_shape")->setVal(0.7); ws2.var("alpha_shape")->setVal(0.7); ws.var("x")->setBin(1); ws2.var("x")->setBin(1); - EXPECT_NEAR(interp->getVal(), interp2->getVal(), 1e-9 * interp->getVal()); + EXPECT_NEAR(sample->getVal(), sample2->getVal(), 1e-9 * sample->getVal()); } TEST(RooFitHS3, HistFactoryConstraintKeyMigration) @@ -2569,7 +2650,8 @@ TEST(RooFitHS3, HistFactoryStatErrorConstraint) // The constraints are exported as one array distribution that the staterror modifier references by name. Doc doc{ws}; EXPECT_FALSE(doc.has("distributions/model_channel0/staterror")); - const std::string modifier = "distributions/model_channel0/samples/sig/modifiers/staterror"; + // The name of the modifier in the document is kept. + const std::string modifier = "distributions/model_channel0/samples/sig/modifiers/mcstat"; EXPECT_FALSE(doc.has(modifier + "/constraint")); const std::vector constraints = doc.strings(modifier + "/constraints"); ASSERT_EQ(constraints.size(), 2u); @@ -2607,8 +2689,7 @@ TEST(RooFitHS3, HistFactoryMissingInterpolation) Imported ws{HistFactoryDoc{"0.2", modifiers}}; EXPECT_EQ(countOccurrences(ws.warnings, "implicit default of older RooFit versions"), 1u) << ws.warnings; - auto *norm = object(ws, "sig_channel0_epsilon"); - EXPECT_EQ(norm->interpolationCodes()[0], 4); + EXPECT_EQ(interpolationCodes(*sigSample(ws), true).at("alpha_norm"), 4); // RooFit always writes a "default_interpolation" if a modifier relies on it. Doc exported{ws}; @@ -3257,16 +3338,17 @@ TEST(RooFitHS3, HistFactoryArrayDistExport) // Opt-out: one constraint distribution per parameter, no array_dist at all. const std::string modifiers = "distributions/model_c1/samples/sig/modifiers/"; + // The staterror modifier, which has no name in the document, keeps the name that the import gave it. + const std::string staterror = modifiers + "c1_sig_syst_2"; Doc plain{ws, {exportArrayDist(false)}}; EXPECT_THAT(plain.json(), Not(HasSubstr("array_dist"))); EXPECT_TRUE(matches(plain, modifiers + "a", R"({"constraints": ["alpha_a_constraint"]})")); - EXPECT_TRUE(matches(plain, modifiers + "staterror", - R"({"constraints": ["gamma_c1_0_constraint", "gamma_c1_1_constraint"]})")); + EXPECT_TRUE(matches(plain, staterror, R"({"constraints": ["gamma_c1_0_constraint", "gamma_c1_1_constraint"]})")); // From the workspace with imported arrays, the components are written on their own as well. Imported ws2{exported}; Doc plainFromArrays{ws2, {exportArrayDist(false)}}; EXPECT_THAT(plainFromArrays.json(), Not(HasSubstr("array_dist"))); - EXPECT_TRUE(matches(plainFromArrays, modifiers + "staterror", + EXPECT_TRUE(matches(plainFromArrays, staterror, R"({"constraints": ["nom_gamma_c1_0_constraint", "nom_gamma_c1_1_constraint"]})")); } @@ -3283,7 +3365,8 @@ TEST(RooFitHS3, TutorialRF515RoundTrip) // The top-level pdf is the RooSimultaneous "main" of the analysis. EXPECT_TRUE(matches(exported, "distributions/main_gaussian_constraints", R"({"type": "array_dist"})")); EXPECT_EQ(exported["distributions/main_gaussian_constraints/components"].num_children(), 3u); - EXPECT_TRUE(matches(exported, "distributions/model_channel1/samples/background1/modifiers/staterror", + // The name of the modifier in the document is kept. + EXPECT_TRUE(matches(exported, "distributions/model_channel1/samples/background1/modifiers/mcstat", R"({"constraints": ["gamma_bin_0_constraint", "gamma_bin_1_constraint"]})")); EXPECT_EQ(reexport(exported), exported.json()); @@ -3497,9 +3580,10 @@ TEST(RooFitHS3, HistFactoryShapeFactor) const std::string json = makeJSON("0.3", freeModifier + "," + constrainedModifier, gammaConstraint("g_c_0")); Imported ws{json}; - // The functions are named as in HistFactory: ShapeFactor for free, ShapeSys for constrained parameters. - EXPECT_NE(ws.function("channel0_free_shapeFactor"), nullptr); - EXPECT_NE(ws.function("channel0_constr_ShapeSys"), nullptr); + // As in HistFactory: ShapeFactor for free, ShapeSys for constrained parameters. + using BinFactorType = RooHistFactorySample::BinFactorType; + EXPECT_EQ(binFactorType(*sigSample(ws), "free"), BinFactorType::ShapeFactor); + EXPECT_EQ(binFactorType(*sigSample(ws), "constr"), BinFactorType::ShapeSys); EXPECT_EQ(countGammaConstraints(ws), 1u); ASSERT_NE(ws.var("g_free_0"), nullptr); EXPECT_FALSE(ws.var("g_free_0")->isConstant()); @@ -3523,13 +3607,13 @@ TEST(RooFitHS3, HistFactoryShapeFactor) Imported ws4{makeJSON("0.3", R"({"type": "shapefactor", "name": "free", "parameters": ["g_free_0", "g_free_1"], "data": {"vals": [0.1, 0.2]}})")}; EXPECT_THAT(ws4.warnings, HasSubstr(R"(the "data.vals" of modifier 'free' of sample 'sig' are ignored)")); - EXPECT_NE(ws4.function("channel0_free_shapeFactor"), nullptr); + EXPECT_EQ(binFactorType(*sigSample(ws4), "free"), BinFactorType::ShapeFactor); EXPECT_EQ(countGammaConstraints(ws4), 0u); // Before HS3 0.3, a "shapesys" with "data.vals" is constrained as before, without deprecation warning. Imported ws5{makeJSON("0.2", R"({"type": "shapesys", "name": "constr", "data": {"vals": [0.1, 0.2]}})")}; EXPECT_THAT(ws5.warnings, Not(HasSubstr("deprecated"))); - EXPECT_NE(ws5.function("channel0_constr_ShapeSys"), nullptr); + EXPECT_EQ(binFactorType(*sigSample(ws5), "constr"), BinFactorType::ShapeSys); EXPECT_EQ(countGammaConstraints(ws5), 2u); } @@ -3540,11 +3624,12 @@ TEST(RooFitHS3, HistFactoryShapeFactorWithoutParameters) RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); Imported ws{HistFactoryDoc{"0.3", R"({"type": "shapefactor", "name": "free", "data": {"vals": [0.1, 0.2]}})"}}; - auto *phf = object(ws, "channel0_free_shapeFactor"); - EXPECT_EQ(phf->paramList().size(), 2u); + EXPECT_EQ(binFactorType(*sigSample(ws), "free"), RooHistFactorySample::BinFactorType::ShapeFactor); + const RooArgList params = binFactorParams(*sigSample(ws), "free"); + EXPECT_EQ(params.size(), 2u); for (const char *name : {"gamma_free_bin_0", "gamma_free_bin_1"}) { auto *gamma = object(ws, name); - EXPECT_NE(phf->paramList().find(*gamma), nullptr) << name; + EXPECT_NE(params.find(*gamma), nullptr) << name; EXPECT_FALSE(gamma->isConstant()) << name; } } @@ -3733,8 +3818,10 @@ TEST(RooFitHS3, HistFactoryShapeFactorSharedBetweenChannels) { Imported ws{ gammasSharedBetweenChannels(R"("type": "shapefactor", "name": "shape")", R"({"contents": [10.0, 20.0]})")}; - EXPECT_NE(ws.function("A_shape_ShapeSys"), nullptr); - EXPECT_NE(ws.function("B_shape_shapeFactor"), nullptr); + EXPECT_EQ(binFactorType(*object(ws, "model_A_bkg_shapes"), "shape"), + RooHistFactorySample::BinFactorType::ShapeSys); + EXPECT_EQ(binFactorType(*object(ws, "model_B_bkg_shapes"), "shape"), + RooHistFactorySample::BinFactorType::ShapeFactor); Doc exported{ws}; EXPECT_TRUE(exported.has("distributions/model_A/samples/0/modifiers/0/constraints")); @@ -3787,7 +3874,7 @@ TEST(RooFitHS3, HistFactoryShapeFactorWithoutName) "constraints": ["g0_constraint", null]})"} .distributions(gammaConstraint("g0")) .sections(defaultDomain({axes({"g0", "g1"}, 0, 5)}))}; - EXPECT_NE(ws.function("channel0_channel0_sig_syst_0_ShapeSys"), nullptr); + EXPECT_EQ(binFactorType(*sigSample(ws), "channel0_sig_syst_0"), RooHistFactorySample::BinFactorType::ShapeSys); } namespace { @@ -3813,8 +3900,7 @@ TEST(RooFitHS3, HistFactoryFunctionGammaWithConstraint) Imported ws{ functionGammaDoc(R"({"type": "staterror", "parameters": ["g0", "g1"], "constraints": ["g0_constraint", null]})") .distributions(gammaConstraint("g0"))}; - auto *phf = object(ws, "mc_stat_channel0"); - RooAbsArg *g0 = phf->paramList().find("g0"); + RooAbsArg *g0 = sigSample(ws)->parameters().find("g0"); ASSERT_NE(g0, nullptr); EXPECT_EQ(dynamic_cast(g0), nullptr); EXPECT_DOUBLE_EQ(static_cast(g0)->getVal(), 1.0); @@ -3880,8 +3966,7 @@ TEST(RooFitHS3, HistFactoryFunctionGammaWithLegacyConstraint) RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); Imported ws{functionGammaDoc(R"({"type": "staterror", "parameters": ["g0", "g1"], "constraint": "Poisson"})")}; - auto *phf = object(ws, "mc_stat_channel0"); - RooAbsArg *g0 = phf->paramList().find("g0"); + RooAbsArg *g0 = sigSample(ws)->parameters().find("g0"); ASSERT_NE(g0, nullptr); EXPECT_EQ(dynamic_cast(g0), nullptr); for (const char *name : {"g0_constraint", "g1_constraint"}) { @@ -3953,8 +4038,12 @@ TEST(RooFitHS3, HistFactoryUnnamedShapeFactorsOfSamples) "model_channel0", {sample("a", contents, R"({"type": "shapefactor", "parameters": ["a0", "a1"]})"), sample("b", contents, R"({"type": "shapefactor", "parameters": ["b0", "b1"]})")})}), defaultDomain({axes({"a0", "a1", "b0", "b1"}, 0, 5)})})}; - EXPECT_NE(ws.function("channel0_channel0_a_syst_0_shapeFactor"), nullptr); - EXPECT_NE(ws.function("channel0_channel0_b_syst_0_shapeFactor"), nullptr); + for (const char *sample : {"a", "b"}) { + auto *hfSample = object(ws, std::string("model_channel0_") + sample + "_shapes"); + EXPECT_EQ(binFactorType(*hfSample, std::string("channel0_") + sample + "_syst_0"), + RooHistFactorySample::BinFactorType::ShapeFactor) + << sample; + } RooAbsPdf *model = ws.pdf("model_channel0"); for (const char *name : {"a0", "a1", "b0", "b1"}) { ASSERT_NE(ws.var(name), nullptr) << name; @@ -4245,6 +4334,10 @@ TEST(RooFitHS3, HistFactoryImportEnablesBinnedLikelihood) // HistFactory exporter split it at the "*" into the factors "1 + 0.2" and "k", which changes the model. TEST(RooFitHS3, FormulaWithTopLevelSumIsNotSplit) { + // The formulas are decomposed for the RooFit objects of the HistFactory models before ROOT 6.44, which the test + // compares with the ones that are read back. + ScopedConfig config{{useHistFactoryPdf(false)}}; + for (std::string expression : {"1 + 0.2 * k", "1.5 - k * 0.2", "2e-1 * k + 1"}) { SCOPED_TRACE(expression); RooWorkspace ws; @@ -4265,10 +4358,12 @@ TEST(RooFitHS3, FormulaWithTopLevelSumIsNotSplit) } // A formula among the factors of a sample is written as one modifier per factor, unless that makes the channel -// invalid: then it stays one custom modifier. +// invalid: then it stays one custom modifier. This is about the RooFit objects of the HistFactory models before ROOT +// 6.44: a RooHistFactoryPdf keeps its custom modifiers as they are. TEST(RooFitHS3, FormulaIsNotSplitIfThatMakesTheChannelInvalid) { RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); + ScopedConfig config{{useHistFactoryPdf(false)}}; const std::string b2 = R"({"name": "b2", "type": "generic_function", "expression": "b*b"})"; const std::string bc = R"({"name": "bc", "type": "generic_function", "expression": "b*c"})"; @@ -4304,6 +4399,10 @@ TEST(RooFitHS3, FormulaIsNotSplitIfThatMakesTheChannelInvalid) // formula is linear and the normsys that it is written as interpolates exponentially. TEST(RooFitHS3, FormulaNormSysVariations) { + // The formulas are decomposed for the RooFit objects of the HistFactory models before ROOT 6.44, which the test + // compares with the ones that are read back. + ScopedConfig config{{useHistFactoryPdf(false)}}; + RooWorkspace ws; importFormulaNormSysChannel(ws); // The import enables the binned likelihood, which the first export doesn't have. @@ -4938,7 +5037,11 @@ TEST(RooFitHS3, HistFactoryExportDoesNotDependOnNames) meas.AddChannel(cr); meas.CollectHistograms(); - std::unique_ptr ws{MakeModelAndMeasurementFast(meas)}; + // The test modifies the RooFit objects of the HistFactory models before ROOT 6.44, which the exporter has to + // recognize by their structure. + HistoToWorkspaceFactoryFast::Configuration cfg; + cfg.useHistFactoryPdf = false; + std::unique_ptr ws{MakeModelAndMeasurementFast(meas, cfg)}; ASSERT_NE(ws, nullptr); auto *mc = static_cast(ws->obj("ModelConfig")); RooAbsData *data = ws->data("obsData"); @@ -5649,14 +5752,14 @@ TEST(RooFitHS3, HistFactoryMultiDimensionalBinOrder) }; Imported ws{makeJSON("0.3")}; - auto *phf = object(ws, "mc_stat_channel0"); + RooHistFactorySample *sample = sigSample(ws); for (int ix = 0; ix < 2; ++ix) { for (int iy = 0; iy < 3; ++iy) { // The constraint of each gamma has the relative uncertainty of its bin. auto *sigma = dynamic_cast(ws.arg((gammaName(ix, iy) + "_sigma").c_str())); ASSERT_NE(sigma, nullptr) << gammaName(ix, iy); EXPECT_NEAR(sigma->getVal(), relError(ix, iy), 1e-12) << gammaName(ix, iy); - // In its bin, the ParamHistFunc takes the value of that gamma. + // In its bin, the sample is scaled with that gamma. ws.var(gammaName(ix, iy))->setVal(1.0 + 0.001 * (1 + 3 * ix + iy)); } } @@ -5664,7 +5767,9 @@ TEST(RooFitHS3, HistFactoryMultiDimensionalBinOrder) for (int iy = 0; iy < 3; ++iy) { ws.var("x")->setBin(ix); ws.var("y")->setBin(iy); - EXPECT_DOUBLE_EQ(phf->getVal(), ws.var(gammaName(ix, iy))->getVal()) << "bin (" << ix << ", " << iy << ")"; + // The value of the sample is a density, and the bins have unit volume. + EXPECT_DOUBLE_EQ(sample->getVal(), content(ix, iy) * ws.var(gammaName(ix, iy))->getVal()) + << "bin (" << ix << ", " << iy << ")"; } } for (int ix = 0; ix < 2; ++ix) { @@ -5676,7 +5781,8 @@ TEST(RooFitHS3, HistFactoryMultiDimensionalBinOrder) // The export writes the gammas in the same order. Doc exported{ws}; EXPECT_TRUE(matches(exported, "distributions/model_channel0", R"({"axes": [{"name": "x"}, {"name": "y"}]})")); - EXPECT_TRUE(matches(exported, "distributions/model_channel0/samples/sig/modifiers/staterror", + // The staterror modifier, which has no name in the document, keeps the name that the import gave it. + EXPECT_TRUE(matches(exported, "distributions/model_channel0/samples/sig/modifiers/channel0_sig_syst_0", R"({"parameters": ["g_0_0", "g_0_1", "g_0_2", "g_1_0", "g_1_1", "g_1_2"]})")); // Documents before HS3 0.3 don't define the order; they are rejected. @@ -7085,7 +7191,8 @@ TEST(RooFitHS3, HistFactoryConstraintSharedWithOtherPdf) Doc exported{ws, {exportArrayDist(arrays)}}; const std::string modifiers = "distributions/" + hfName + "/samples/signal/modifiers/"; EXPECT_TRUE(matches(exported, "distributions/shared_constraint", R"({"type": "gaussian_dist"})")); - EXPECT_TRUE(matches(exported, modifiers + "theta", R"({"constraints": ["shared_constraint"]})")); + // The modifiers keep their names in the document. + EXPECT_TRUE(matches(exported, modifiers + "shared", R"({"constraints": ["shared_constraint"]})")); // The product only keeps the factor that is not a term of the likelihood. EXPECT_TRUE(matches(exported, "distributions/" + otherName, R"({"factors": ["shape"]})")); // The constraint that only the HistFactory channel uses is still collected. @@ -7094,7 +7201,7 @@ TEST(RooFitHS3, HistFactoryConstraintSharedWithOtherPdf) EXPECT_EQ(exported.has("distributions/" + array), arrays); if (arrays) { EXPECT_TRUE(matches(exported, "distributions/" + array, R"({"components": [{"x": "eta_obs"}]})")); - EXPECT_TRUE(matches(exported, modifiers + "eta", R"({"constraints": [")" + array + R"("]})")); + EXPECT_TRUE(matches(exported, modifiers + "own", R"({"constraints": [")" + array + R"("]})")); } // Valid HS3 0.3: read back without warnings. @@ -7188,7 +7295,7 @@ TEST(RooFitHS3, HistFactoryConstraintAlsoExternal) Doc exported{ws}; EXPECT_TRUE(matches(exported, "distributions/constraint", R"({"type": "gaussian_dist"})")); EXPECT_TRUE( - matches(exported, "distributions/HF/samples/signal/modifiers/theta", R"({"constraints": ["constraint"]})")); + matches(exported, "distributions/HF/samples/signal/modifiers/syst", R"({"constraints": ["constraint"]})")); EXPECT_TRUE(matches(exported, "likelihoods/0", R"({"aux_distributions": ["constraint"]})")); } diff --git a/roofit/man/man1/hist2workspace.1 b/roofit/man/man1/hist2workspace.1 index 2bcee0e5aa4cf..aa603a7cd8f31 100644 --- a/roofit/man/man1/hist2workspace.1 +++ b/roofit/man/man1/hist2workspace.1 @@ -1,6 +1,6 @@ .TH hist2workspace 1 .SH SYNOPSIS -usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] +usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] [-legacy_model_structure] .SH DESCRIPTION hist2workspace is a utility to create RooFit/RooStats workspace from histograms @@ -13,3 +13,5 @@ switch HistFactory message stream to INFO level switch HistFactory message stream to DEBUG level .IP -disable_binned_fit_optimization disable the binned fit optimization used in HistFactory since ROOT 6.28 +.IP -legacy_model_structure +build the channels from several RooFit objects per histogram, as before ROOT 6.44 diff --git a/roofit/roofitcore/CMakeLists.txt b/roofit/roofitcore/CMakeLists.txt index af0f2340177b3..e1f917a151e3b 100644 --- a/roofit/roofitcore/CMakeLists.txt +++ b/roofit/roofitcore/CMakeLists.txt @@ -121,6 +121,7 @@ set(ROOFITCORE_HEADERS RooFit/Detail/NormalizationHelpers.h RooFit/Detail/RooNLLVarNew.h RooFit/Detail/RooNormalizedPdf.h + RooFit/Detail/WorkspaceStreamingContext.h RooFit/EvalContext.h RooFit/Evaluator.h RooFit/Floats.h @@ -443,6 +444,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitCore src/TestStatistics/RooUnbinnedL.cxx src/TestStatistics/buildLikelihood.cxx src/TestStatistics/SharedOffset.cxx + src/WorkspaceStreamingContext.cxx ${RooFitMPTestStatisticsSources} DICTIONARY_OPTIONS "-writeEmptyRootPCM" diff --git a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h index 7cc6ab7086791..f5237441f9f8b 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h +++ b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h @@ -371,6 +371,76 @@ double flexibleInterp(unsigned int code, ParamsArray params, unsigned int n, Dou return doCutoff && total <= 0 ? TMath::Limits::Min() : total; } +/// The scale factor of a sample of a HistFactory model (RooHistFactorySample): the product of the NormSys factors, +/// interpolated like in FlexibleInterpVar, and of the NormFactors. +template +double +histFactoryScale(ParamsArray params, unsigned int nNormSys, int const *normSysParams, int const *normSysCodes, + DoubleArray normSysLo, DoubleArray normSysHi, unsigned int nNormFactors, int const *normFactorParams) +{ + double coef = 1.0; + if (nNormSys > 0) { + double total = 1.0; + for (unsigned int i = 0; i < nNormSys; ++i) { + // Code 4 of FlexibleInterpVar is code 5 of PiecewiseInterpolation. + const unsigned int code = normSysCodes[i] == 4 ? 5 : normSysCodes[i]; + total += flexibleInterpSingle(code, normSysLo[i], normSysHi[i], 1.0, 1.0, params[normSysParams[i]], total); + } + if (total <= 0) { + total = TMath::Limits::Min(); + } + coef *= total; + } + for (unsigned int i = 0; i < nNormFactors; ++i) { + coef *= params[normFactorParams[i]]; + } + return coef; +} + +/// The content of a bin of a sample of a HistFactory model (RooHistFactorySample) without the scale factor: the +/// nominal content with the HistoSys variations, interpolated like in PiecewiseInterpolation and clamped at zero, +/// times the parameters of the bin-wise modifiers. +template +double +histFactoryBin(ParamsArray params, unsigned int iBin, unsigned int nBins, DoubleArray nominal, unsigned int nHistoSys, + int const *histoSysParams, int const *histoSysCodes, DoubleArray histoSysLo, DoubleArray histoSysHi, + unsigned int nBinFactors, int const *binFactorParams) +{ + const double nom = nominal[iBin]; + double value = nom; + if (nHistoSys > 0) { + double sum = nom; + for (unsigned int i = 0; i < nHistoSys; ++i) { + sum += flexibleInterpSingle(histoSysCodes[i], histoSysLo[i * nBins + iBin], histoSysHi[i * nBins + iBin], 1.0, + nom, params[histoSysParams[i]], sum); + } + value = sum < 0 ? 0 : sum; + } + for (unsigned int i = 0; i < nBinFactors; ++i) { + value *= params[binFactorParams[i * nBins + iBin]]; + } + return value; +} + +/// The weighted sum of the bins of a sample of a HistFactory model (RooHistFactorySample) without the scale factor. +/// With the fractions of the bins in the integration range as weights, this is the integral of the sample. +template +double +histFactoryIntegral(ParamsArray params, DoubleArray weights, unsigned int nBins, DoubleArray nominal, + unsigned int nHistoSys, int const *histoSysParams, int const *histoSysCodes, DoubleArray histoSysLo, + DoubleArray histoSysHi, unsigned int nBinFactors, int const *binFactorParams) +{ + double out = 0.0; + for (unsigned int i = 0; i < nBins; ++i) { + if (weights[i] != 0.0) { + out += histFactoryBin(params, i, nBins, nominal, nHistoSys, histoSysParams, histoSysCodes, histoSysLo, + histoSysHi, nBinFactors, binFactorParams) * + weights[i]; + } + } + return out; +} + inline double landau(double x, double mu, double sigma) { if (sigma <= 0.) diff --git a/roofit/roofitcore/inc/RooFit/Detail/WorkspaceStreamingContext.h b/roofit/roofitcore/inc/RooFit/Detail/WorkspaceStreamingContext.h new file mode 100644 index 0000000000000..97a7d1d164896 --- /dev/null +++ b/roofit/roofitcore/inc/RooFit/Detail/WorkspaceStreamingContext.h @@ -0,0 +1,69 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_Detail_WorkspaceStreamingContext_h +#define RooFit_Detail_WorkspaceStreamingContext_h + +#include +#include +#include +#include + +class RooWorkspace; +class TFile; + +namespace RooFit { +namespace Detail { + +/// State shared by the objects that are streamed as part of one RooWorkspace, for as long as the workspace is +/// written or read. Classes with large payloads use it to store them outside of the workspace buffer, which is +/// limited to 1 GB. RooWorkspace::Write() creates the context for writing, and RooWorkspace::Streamer() for reading. +class WorkspaceStreamingContext { +public: + /// State that one class keeps while the workspace is streamed. + class Extension { + public: + virtual ~Extension() = default; + /// Called after all objects of the workspace were streamed. When writing, the workspace is then already in the + /// file. + virtual void finalize() {} + }; + + WorkspaceStreamingContext(RooWorkspace const &workspace, TFile *file); + ~WorkspaceStreamingContext(); + + WorkspaceStreamingContext(WorkspaceStreamingContext const &) = delete; + WorkspaceStreamingContext &operator=(WorkspaceStreamingContext const &) = delete; + + /// The context of the workspace that this thread is streaming, or nullptr. + static WorkspaceStreamingContext *current(); + + RooWorkspace const &workspace() const { return _workspace; } + /// The file that the workspace is written to, if objects can write their payloads to it, else nullptr. + TFile *file() const { return _file; } + + Extension *extension(std::string const &name) const; + Extension &addExtension(std::string const &name, std::unique_ptr extension); + + void finalize(); + +private: + RooWorkspace const &_workspace; + TFile *_file = nullptr; + WorkspaceStreamingContext *_previous = nullptr; + std::vector>> _extensions; +}; + +} // namespace Detail +} // namespace RooFit + +#endif diff --git a/roofit/roofitcore/inc/RooWorkspace.h b/roofit/roofitcore/inc/RooWorkspace.h index 6f84ff97eaac9..d7640ca146cbb 100644 --- a/roofit/roofitcore/inc/RooWorkspace.h +++ b/roofit/roofitcore/inc/RooWorkspace.h @@ -53,6 +53,9 @@ class RooWorkspace : public TNamed { TObject *Clone(const char *newname="") const override; + Int_t Write(const char *name = nullptr, Int_t option = 0, Int_t bufsize = 0) const override; + Int_t Write(const char *name = nullptr, Int_t option = 0, Int_t bufsize = 0) override; + bool importClassCode(const char* pat="*", bool doReplace=false) ; bool importClassCode(TClass* theClass, bool doReplace=false) ; diff --git a/roofit/roofitcore/src/RooWorkspace.cxx b/roofit/roofitcore/src/RooWorkspace.cxx index 899702865e60b..7c5802559f5e4 100644 --- a/roofit/roofitcore/src/RooWorkspace.cxx +++ b/roofit/roofitcore/src/RooWorkspace.cxx @@ -62,6 +62,7 @@ and try reading again. #include #include #include +#include #include "TBuffer.h" #include "TInterpreter.h" @@ -2530,7 +2531,9 @@ void RooWorkspace::Streamer(TBuffer &R__b) { if (R__b.IsReading()) { + RooFit::Detail::WorkspaceStreamingContext context{*this, nullptr}; R__b.ReadClassBuffer(RooWorkspace::Class(), this); + context.finalize(); // Perform any pass-2 schema evolution here for (RooAbsArg *node : _allOwnedNodes) { @@ -2631,6 +2634,31 @@ void RooWorkspace::Streamer(TBuffer &R__b) } } +//////////////////////////////////////////////////////////////////////////////// +/// Write the workspace to the current directory, like TObject::Write(). +/// +/// Objects in the workspace can store large payloads next to the workspace in the file, after the workspace itself +/// was written. HistFactory models do that with their histograms, so that the workspace is not limited by the 1 GB +/// size limit of single objects in ROOT files. This only happens when writing with this function or with +/// writeToFile(), not with TDirectory::WriteObject() or TDirectory::WriteTObject(), which write everything into the +/// workspace object. HistFactory writes an RNTuple, which also writes the other objects in memory that are attached to +/// the file, like TFile::Write(). + +Int_t RooWorkspace::Write(const char *name, Int_t option, Int_t bufsize) const +{ + TFile *file = gDirectory ? gDirectory->GetFile() : nullptr; + const bool canWritePayloads = file && file->IsWritable() && file->IsBinary(); + RooFit::Detail::WorkspaceStreamingContext context{*this, canWritePayloads ? file : nullptr}; + const Int_t nbytes = TObject::Write(name, option, bufsize); + context.finalize(); + return nbytes; +} + +Int_t RooWorkspace::Write(const char *name, Int_t option, Int_t bufsize) +{ + return static_cast(this)->Write(name, option, bufsize); +} + diff --git a/roofit/roofitcore/src/WorkspaceStreamingContext.cxx b/roofit/roofitcore/src/WorkspaceStreamingContext.cxx new file mode 100644 index 0000000000000..55887500edb34 --- /dev/null +++ b/roofit/roofitcore/src/WorkspaceStreamingContext.cxx @@ -0,0 +1,72 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include + +#include + +namespace { + +thread_local RooFit::Detail::WorkspaceStreamingContext *currentContext = nullptr; + +} // namespace + +namespace RooFit { +namespace Detail { + +WorkspaceStreamingContext::WorkspaceStreamingContext(RooWorkspace const &workspace, TFile *file) + : _workspace{workspace}, _file{file}, _previous{currentContext} +{ + currentContext = this; +} + +WorkspaceStreamingContext::~WorkspaceStreamingContext() +{ + currentContext = _previous; +} + +WorkspaceStreamingContext *WorkspaceStreamingContext::current() +{ + return currentContext; +} + +/// The extension with a given name, or nullptr if there is none. +WorkspaceStreamingContext::Extension *WorkspaceStreamingContext::extension(std::string const &name) const +{ + for (auto const &item : _extensions) { + if (item.first == name) { + return item.second.get(); + } + } + return nullptr; +} + +WorkspaceStreamingContext::Extension & +WorkspaceStreamingContext::addExtension(std::string const &name, std::unique_ptr extension) +{ + if (this->extension(name)) { + throw std::invalid_argument("WorkspaceStreamingContext: there is already an extension named \"" + name + "\""); + } + _extensions.emplace_back(name, std::move(extension)); + return *_extensions.back().second; +} + +/// Finalize the extensions in the order in which they were added. +void WorkspaceStreamingContext::finalize() +{ + for (auto const &item : _extensions) { + item.second->finalize(); + } +} + +} // namespace Detail +} // namespace RooFit