From 9adc06e2a18369f35ef8cad05bcf1ddf02b72483 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 16 Nov 2026 15:36:22 +0000 Subject: [PATCH 1/5] [RF] Add RooHistFactoryPdf and RooHistFactorySample A RooHistFactorySample is one sample of a HistFactory channel: the nominal histogram with all its modifiers, in a single object. Its histogram data is shared between clones. A RooHistFactoryPdf is the channel, a RooRealSumPdf of its samples, and corresponds to the HS3 histfactory_dist. The arithmetic is in RooFit::Detail::MathFuncs, so that the evaluation and code generation use the same functions, in the order of operations of the HistFactory models so far. Assisted-by: claude-code:claude-opus-5-5 --- roofit/codegen/inc/RooFit/CodegenImpl.h | 3 + roofit/codegen/src/CodegenImpl.cxx | 103 ++++ roofit/histfactory/CMakeLists.txt | 4 + roofit/histfactory/inc/LinkDef.h | 3 + .../RooStats/HistFactory/RooHistFactoryPdf.h | 39 ++ .../HistFactory/RooHistFactorySample.h | 171 ++++++ roofit/histfactory/src/RooHistFactoryPdf.cxx | 111 ++++ .../histfactory/src/RooHistFactorySample.cxx | 538 ++++++++++++++++++ .../roofitcore/inc/RooFit/Detail/MathFuncs.h | 70 +++ 9 files changed, 1042 insertions(+) create mode 100644 roofit/histfactory/inc/RooStats/HistFactory/RooHistFactoryPdf.h create mode 100644 roofit/histfactory/inc/RooStats/HistFactory/RooHistFactorySample.h create mode 100644 roofit/histfactory/src/RooHistFactoryPdf.cxx create mode 100644 roofit/histfactory/src/RooHistFactorySample.cxx diff --git a/roofit/codegen/inc/RooFit/CodegenImpl.h b/roofit/codegen/inc/RooFit/CodegenImpl.h index eca7bad066cb2..b954201a51317 100644 --- a/roofit/codegen/inc/RooFit/CodegenImpl.h +++ b/roofit/codegen/inc/RooFit/CodegenImpl.h @@ -42,6 +42,7 @@ class RooFunctorPdfBinding; class RooGamma; class RooGaussian; class RooGenericPdf; +class RooHistFactorySample; class RooHistFunc; class RooHistPdf; class RooLandau; @@ -109,6 +110,7 @@ void codegenImpl(RooFunctorPdfBinding &arg, CodegenContext &ctx); void codegenImpl(RooGamma &arg, CodegenContext &ctx); void codegenImpl(RooGaussian &arg, CodegenContext &ctx); void codegenImpl(RooGenericPdf &arg, CodegenContext &ctx); +void codegenImpl(RooHistFactorySample &arg, CodegenContext &ctx); void codegenImpl(RooHistFunc &arg, CodegenContext &ctx); void codegenImpl(RooHistPdf &arg, CodegenContext &ctx); void codegenImpl(RooLandau &arg, CodegenContext &ctx); @@ -159,6 +161,7 @@ std::string codegenIntegralImpl(RooEfficiency &arg, int code, const char *rangeN std::string codegenIntegralImpl(RooExponential &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooGamma &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooGaussian &arg, int code, const char *rangeName, CodegenContext &ctx); +std::string codegenIntegralImpl(RooHistFactorySample &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooHistFunc &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooHistPdf &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooLandau &arg, int code, const char *rangeName, CodegenContext &ctx); diff --git a/roofit/codegen/src/CodegenImpl.cxx b/roofit/codegen/src/CodegenImpl.cxx index 1150f1ab5b2de..6b13889d0abd9 100644 --- a/roofit/codegen/src/CodegenImpl.cxx +++ b/roofit/codegen/src/CodegenImpl.cxx @@ -59,6 +59,7 @@ #include #include #include +#include #include #include @@ -594,6 +595,108 @@ void codegenImpl(RooGenericPdf &arg, CodegenContext &ctx) ctx.addResult(&arg, funcName + "(" + inputVar + ")"); } +namespace { + +/// The array arguments of the kernels of a RooHistFactorySample. +std::string intArrayArg(CodegenContext &ctx, std::vector const &arr) +{ + return arr.empty() ? "nullptr" : ctx.buildArg(std::span{arr}); +} + +std::string doubleArrayArg(CodegenContext &ctx, std::vector const &arr) +{ + return ctx.buildArg(std::span{arr}); +} + +/// The arguments of RooFit::Detail::MathFuncs::histFactoryBin() and histFactoryIntegral() after the bin. +std::string histFactoryBinArgs(RooHistFactorySample const &arg, CodegenContext &ctx) +{ + auto const &data = arg.data(); + return std::to_string(data.numBins()) + ", " + doubleArrayArg(ctx, data.contents) + ", " + + std::to_string(data.histoSysParams.size()) + ", " + intArrayArg(ctx, data.histoSysParams) + ", " + + intArrayArg(ctx, data.histoSysCodes) + ", " + doubleArrayArg(ctx, data.histoSysLo) + ", " + + doubleArrayArg(ctx, data.histoSysHi) + ", " + std::to_string(data.binFactorNames.size()) + ", " + + intArrayArg(ctx, data.binFactorParams); +} + +/// The scale factor of a RooHistFactorySample, which only depends on the parameters and is therefore computed outside +/// of the loop over the data. +std::string histFactoryScale(RooHistFactorySample const &arg, std::string const ¶ms, CodegenContext &ctx) +{ + auto const &data = arg.data(); + std::string scaleName = ctx.getTmpVarName(); + std::string call = "RooFit::Detail::MathFuncs::histFactoryScale(" + params + ", " + + std::to_string(data.normSysParams.size()) + ", " + intArrayArg(ctx, data.normSysParams) + ", " + + intArrayArg(ctx, data.normSysCodes) + ", " + doubleArrayArg(ctx, data.normSysLo) + ", " + + doubleArrayArg(ctx, data.normSysHi) + ", " + std::to_string(data.normFactorParams.size()) + ", " + + intArrayArg(ctx, data.normFactorParams) + ")"; + ctx.addToCodeBody("double " + scaleName + " = " + call + ";\n", /*isScopeIndep=*/true); + return scaleName; +} + +} // namespace + +void codegenImpl(RooHistFactorySample &arg, CodegenContext &ctx) +{ + auto const &data = arg.data(); + std::string const ¶ms = ctx.buildArg(arg.parameters()); + const std::string scale = arg.parameters().empty() ? "1.0" : histFactoryScale(arg, params, ctx); + + // The index of the bin with the current values of the observables. The last + // observable varies fastest. + std::string idx; + unsigned int stride = 1; + for (int iDim = static_cast(data.edges.size()) - 1; iDim >= 0; --iDim) { + std::vector const &edges = data.edges[iDim]; + const unsigned int nBinsDim = edges.size() - 1; + RooAbsArg const &obs = arg.observables()[iDim]; + // The bin numbers are computed with a coefficient of one and only then + // multiplied with the stride, because uniformBinNumber() would + // multiply with the coefficient before rounding down. + std::string term = + data.uniform[iDim] + ? ctx.buildCall(mathFunc("uniformBinNumber"), edges.front(), edges.back(), obs, nBinsDim, 1.0) + : ctx.buildCall(mathFunc("binNumber"), obs, 1.0, std::span{edges}, edges.size(), nBinsDim - 1, + 0); + if (stride > 1) { + term = std::to_string(stride) + " * " + term; + } + idx = idx.empty() ? term : term + " + " + idx; + stride *= nBinsDim; + } + std::string idxName = ctx.getTmpVarName(); + ctx.addToCodeBody(&arg, "unsigned int " + idxName + " = " + idx + ";\n"); + + std::string value = mathFunc("histFactoryBin") + "(" + (arg.parameters().empty() ? "nullptr" : params) + ", " + + idxName + ", " + histFactoryBinArgs(arg, ctx) + ")"; + if (!arg.yieldsMode()) { + std::vector invVolumes(data.numBins()); + for (std::size_t i = 0; i < invVolumes.size(); ++i) { + invVolumes[i] = 1. / arg.binVolume(i); + } + value = "(" + value + " * (" + doubleArrayArg(ctx, invVolumes) + ")[" + idxName + "])"; + } + ctx.addResult(&arg, value + " * " + scale); +} + +std::string codegenIntegralImpl(RooHistFactorySample &arg, int code, const char *rangeName, CodegenContext &ctx) +{ + // Only integrals over all observables are supported, because otherwise + // the integral depends on the bins of the other observables. + if (code != (1 << arg.observables().size())) { + std::stringstream errorMsg; + errorMsg << "codegen: integrals of RooHistFactorySample \"" << arg.GetName() + << "\" over only some of its observables are not supported"; + oocoutE(&arg, InputArguments) << errorMsg.str() << std::endl; + throw std::runtime_error(errorMsg.str()); + } + std::string const ¶ms = ctx.buildArg(arg.parameters()); + const std::string scale = arg.parameters().empty() ? "1.0" : histFactoryScale(arg, params, ctx); + return "(" + mathFunc("histFactoryIntegral") + "(" + (arg.parameters().empty() ? "nullptr" : params) + ", " + + doubleArrayArg(ctx, arg.integralWeights(code, rangeName)) + ", " + histFactoryBinArgs(arg, ctx) + ") * " + + scale + ")"; +} + void codegenImpl(RooHistFunc &arg, CodegenContext &ctx) { rooHistTranslateImpl(arg, ctx, arg.getInterpolationOrder(), arg.dataHist(), arg.variables(), false, diff --git a/roofit/histfactory/CMakeLists.txt b/roofit/histfactory/CMakeLists.txt index 12794897e76ed..08f9b99c6c09b 100644 --- a/roofit/histfactory/CMakeLists.txt +++ b/roofit/histfactory/CMakeLists.txt @@ -35,6 +35,8 @@ set(HISTFACTORY_HEADERS RooStats/HistFactory/ParamHistFunc.h RooStats/HistFactory/PiecewiseInterpolation.h RooStats/HistFactory/RooBarlowBeestonLL.h + RooStats/HistFactory/RooHistFactoryPdf.h + RooStats/HistFactory/RooHistFactorySample.h ) ROOT_STANDARD_LIBRARY_PACKAGE(HistFactory @@ -53,6 +55,8 @@ ROOT_STANDARD_LIBRARY_PACKAGE(HistFactory src/ParamHistFunc.cxx src/PiecewiseInterpolation.cxx src/RooBarlowBeestonLL.cxx + src/RooHistFactoryPdf.cxx + src/RooHistFactorySample.cxx ${HISTFACTORY_XML_SOURCES} DICTIONARY_OPTIONS "-writeEmptyRootPCM" diff --git a/roofit/histfactory/inc/LinkDef.h b/roofit/histfactory/inc/LinkDef.h index d3ca88cb74a9b..06ceb39674f09 100644 --- a/roofit/histfactory/inc/LinkDef.h +++ b/roofit/histfactory/inc/LinkDef.h @@ -9,6 +9,9 @@ #pragma link C++ class PiecewiseInterpolation- ; #pragma link C++ class ParamHistFunc+ ; +#pragma link C++ class RooHistFactoryPdf+ ; +#pragma link C++ class RooHistFactorySample- ; +#pragma link C++ class RooHistFactorySample::Data+ ; #pragma link C++ class RooStats::HistFactory::LinInterpVar+ ; #pragma link C++ class RooStats::HistFactory::FlexibleInterpVar+ ; #pragma link C++ class RooStats::HistFactory::HistoToWorkspaceFactoryFast+ ; diff --git a/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactoryPdf.h b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactoryPdf.h new file mode 100644 index 0000000000000..0e04a0297e184 --- /dev/null +++ b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactoryPdf.h @@ -0,0 +1,39 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooStats_HistFactory_RooHistFactoryPdf_h +#define RooStats_HistFactory_RooHistFactoryPdf_h + +#include + +#include + +class RooHistFactorySample; + +class RooHistFactoryPdf : public RooRealSumPdf { +public: + RooHistFactoryPdf() = default; + RooHistFactoryPdf(const char *name, const char *title, RooArgList const &samples); + RooHistFactoryPdf(const RooHistFactoryPdf &other, const char *name = nullptr); + TObject *clone(const char *newname = nullptr) const override { return new RooHistFactoryPdf(*this, newname); } + + std::size_t numSamples() const { return _funcList.size(); } + RooHistFactorySample const &sample(std::size_t i) const; + RooHistFactorySample const *sample(std::string const &sampleName) const; + RooArgList const &observables() const; + + void printMetaArgs(std::ostream &os) const override; + + ClassDefOverride(RooHistFactoryPdf, 1); +}; + +#endif diff --git a/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactorySample.h b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactorySample.h new file mode 100644 index 0000000000000..d78a9d92a7a7e --- /dev/null +++ b/roofit/histfactory/inc/RooStats/HistFactory/RooHistFactorySample.h @@ -0,0 +1,171 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooStats_HistFactory_RooHistFactorySample_h +#define RooStats_HistFactory_RooHistFactorySample_h + +#include +#include + +#include + +#include +#include +#include + +class RooHistFactorySample : public RooAbsReal { +public: + /// The types of modifiers that scale each bin with its own parameter. + enum class BinFactorType { + StatError = 0, + ShapeSys = 1, + ShapeFactor = 2 + }; + + /// Scales the sample with a parameter, or with a function of the parameters (a "custom" modifier in HS3). + struct NormFactor { + std::string name; + RooAbsReal *param = nullptr; + }; + + /// Scales the sample with a factor that is interpolated between `lo` and `hi`. The interpolation codes are the ones + /// of FlexibleInterpVar. + struct NormSys { + std::string name; + RooAbsReal *param = nullptr; + double lo = 1.0; + double hi = 1.0; + int interpCode = 4; + }; + + /// Interpolates the bin contents between the `lo` and `hi` variations. The interpolation codes are the ones of + /// PiecewiseInterpolation. + struct HistoSys { + std::string name; + RooAbsReal *param = nullptr; + std::vector lo; + std::vector hi; + int interpCode = 4; + }; + + /// Scales each bin with its own parameter, given in the order of the bins. + struct BinFactor { + std::string name; + BinFactorType type = BinFactorType::ShapeFactor; + RooArgList params; + }; + + /// Everything that defines a sample. The bins are ordered like in RooDataHist and HS3: the last observable varies + /// fastest. + struct Spec { + std::string sampleName; + std::vector contents; + std::vector errors; ///< Absolute statistical uncertainties, optional. + std::vector normFactors; + std::vector normSys; + std::vector histoSys; + std::vector binFactors; + }; + + /// The immutable data of a sample, shared between the clones of a sample. + struct Data { + std::string sampleName; + std::vector> edges; ///< The bin boundaries of each observable. + std::vector uniform; ///< If the binning of an observable is uniform. + std::vector contents; + std::vector errors; + + std::vector normFactorNames; + std::vector normFactorParams; + + std::vector normSysNames; + std::vector normSysParams; + std::vector normSysLo; + std::vector normSysHi; + std::vector normSysCodes; + + std::vector histoSysNames; + std::vector histoSysParams; + std::vector histoSysCodes; + std::vector histoSysLo; ///< One variation after the other. + std::vector histoSysHi; ///< One variation after the other. + + std::vector binFactorNames; + std::vector binFactorTypes; + std::vector binFactorParams; ///< One modifier after the other, one parameter for each bin. + + std::size_t numBins() const { return contents.size(); } + + ClassDefNV(Data, 1); + }; + + RooHistFactorySample() = default; + RooHistFactorySample(const char *name, const char *title, RooArgList const &observables, Spec const &spec); + RooHistFactorySample(const RooHistFactorySample &other, const char *name = nullptr); + TObject *clone(const char *newname = nullptr) const override { return new RooHistFactorySample(*this, newname); } + + std::string const &sampleName() const { return _data->sampleName; } + RooArgList const &observables() const { return _obs; } + RooArgList const ¶meters() const { return _params; } + Data const &data() const { return *_data; } + std::size_t numBins() const { return _data->numBins(); } + + double binVolume(std::size_t iBin) const; + void binValues(std::span out, bool density) const; + + void setHistoSysInterpCode(RooAbsReal const ¶m, int code); + void setNormSysInterpCode(RooAbsReal const ¶m, int code); + + bool isBinnedDistribution(const RooArgSet &obs) const override; + std::list *binBoundaries(RooAbsRealLValue &obs, double xlo, double xhi) const override; + std::list *plotSamplingHint(RooAbsRealLValue &obs, double xlo, double xhi) const override; + + Int_t getAnalyticalIntegral(RooArgSet &allVars, RooArgSet &analVars, const char *rangeName = nullptr) const override; + double analyticalIntegral(Int_t code, const char *rangeName = nullptr) const override; + std::vector integralWeights(Int_t code, const char *rangeName = nullptr) const; + + std::unique_ptr + compileForNormSet(RooArgSet const &normSet, RooFit::Detail::CompileContext &ctx) const override; + + /// If the values are the yields in the bins, which is the case in the binned likelihood. Otherwise, they are the + /// densities, i.e. the yields divided by the bin volumes. + bool yieldsMode() const { return _yieldsMode; } + + void printMetaArgs(std::ostream &os) const override; + +protected: + double evaluate() const override; + void doEval(RooFit::EvalContext &ctx) const override; + +private: + int observableIndex(RooAbsArg const &obs) const; + std::size_t binIndex(std::span values) const; + double binValue(double const *params, std::size_t iBin) const; + double scaleFactor(double const *params) const; + template + void fillParams(ParamValue &¶mValue) const; + void setInterpCode(std::vector Data::*codes, std::vector Data::*params, RooAbsReal const ¶m, int code, + int maxCode, const char *what); + + RooListProxy _obs; + RooListProxy _params; + std::shared_ptr _data; /// _paramVals; /// _binVals; /// _obsVals; /// + +#include + +#include +#include + +#include + +namespace { + +RooArgList unitCoefficients(std::size_t n) +{ + RooArgList coefs; + for (std::size_t i = 0; i < n; ++i) { + coefs.add(RooFit::RooConst(1.0)); + } + return coefs; +} + +void checkSamples(const char *name, RooArgList const &samples) +{ + if (samples.empty()) { + throw std::invalid_argument(std::string("RooHistFactoryPdf \"") + name + "\": there are no samples"); + } + RooHistFactorySample const *first = nullptr; + for (RooAbsArg *arg : samples) { + auto *sample = dynamic_cast(arg); + if (!sample) { + throw std::invalid_argument(std::string("RooHistFactoryPdf \"") + name + "\": \"" + arg->GetName() + + "\" is not a RooHistFactorySample"); + } + if (!first) { + first = sample; + continue; + } + bool same = + sample->observables().size() == first->observables().size() && sample->data().edges == first->data().edges; + for (std::size_t i = 0; same && i < first->observables().size(); ++i) { + same = &sample->observables()[i] == &first->observables()[i]; + } + if (!same) { + throw std::invalid_argument(std::string("RooHistFactoryPdf \"") + name + "\": the sample \"" + + sample->GetName() + "\" has other observables or bins than the sample \"" + + first->GetName() + "\""); + } + } +} + +} // namespace + +/// Create a HistFactory channel from its samples, which need to have the same observables and bins. +RooHistFactoryPdf::RooHistFactoryPdf(const char *name, const char *title, RooArgList const &samples) + : RooRealSumPdf(name, title, (checkSamples(name, samples), samples), unitCoefficients(samples.size()), true) +{ +} + +RooHistFactoryPdf::RooHistFactoryPdf(const RooHistFactoryPdf &other, const char *name) : RooRealSumPdf(other, name) {} + +RooHistFactorySample const &RooHistFactoryPdf::sample(std::size_t i) const +{ + return static_cast(_funcList[i]); +} + +/// The sample with a given name in the HistFactory model, or nullptr if there is none. +RooHistFactorySample const *RooHistFactoryPdf::sample(std::string const &sampleName) const +{ + for (std::size_t i = 0; i < numSamples(); ++i) { + if (sample(i).sampleName() == sampleName) { + return &sample(i); + } + } + return nullptr; +} + +RooArgList const &RooHistFactoryPdf::observables() const +{ + return sample(0).observables(); +} + +void RooHistFactoryPdf::printMetaArgs(std::ostream &os) const +{ + os << "samples=("; + for (std::size_t i = 0; i < numSamples(); ++i) { + os << (i > 0 ? "," : "") << sample(i).GetName(); + } + os << ") "; +} diff --git a/roofit/histfactory/src/RooHistFactorySample.cxx b/roofit/histfactory/src/RooHistFactorySample.cxx new file mode 100644 index 0000000000000..14bdfdf820f2d --- /dev/null +++ b/roofit/histfactory/src/RooHistFactorySample.cxx @@ -0,0 +1,538 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +/** \class RooHistFactorySample +\ingroup HistFactory + +One sample of a HistFactory channel, i.e. of a RooHistFactoryPdf: a histogram whose bin contents are modified by +parameters. It corresponds to a sample of the `histfactory_dist` in the HS3 standard. The value in bin \f$ b \f$ is + +\f[ + \nu_b = \left( c_b + \sum_i I_i(\alpha_i; l_{i,b}, c_b, h_{i,b}) \right) + \prod_j \gamma_{j,b} \; \prod_k \eta_k(\theta_k) \prod_m \mu_m , +\f] + +with the nominal contents \f$ c_b \f$, the HistoSys variations \f$ l_{i,b} \f$ and \f$ h_{i,b} \f$, the parameters +\f$ \gamma_{j,b} \f$ of the bin-wise modifiers (staterror, shapesys, shapefactor), the NormSys factors +\f$ \eta_k \f$ and the NormFactors \f$ \mu_m \f$. The interpolations are the ones of PiecewiseInterpolation (HistoSys) +and FlexibleInterpVar (NormSys), and if there is a HistoSys, the interpolated bin contents are clamped at zero. + +The value of the function is the density in the bin of the current observable values, i.e. \f$ \nu_b \f$ divided by +the bin volume. In the binned likelihood, the sample is evaluated as the yield \f$ \nu_b \f$ itself. + +In contrast to the HistFactory models before ROOT 6.44, which consist of several RooFit objects for each histogram, +a sample is a single object. The histogram data is shared between the copies of a sample. +**/ + +#include + +#include "HistFactoryInterpolationCodeUtils.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace { + +int checkedIndex(RooArgList ¶ms, RooAbsReal *param, std::string const &sample, std::string const &modifier) +{ + if (!param) { + throw std::invalid_argument("RooHistFactorySample: the modifier \"" + modifier + "\" of sample \"" + sample + + "\" has no parameter"); + } + int index = params.index(param); + if (index < 0) { + params.add(*param); + index = params.size() - 1; + } + return index; +} + +void checkSize(std::size_t size, std::size_t nBins, std::string const &sample, std::string const &what) +{ + if (size != nBins) { + throw std::invalid_argument("RooHistFactorySample: " + what + " of sample \"" + sample + "\" has " + + std::to_string(size) + " bins instead of " + std::to_string(nBins)); + } +} + +double binWidth(RooHistFactorySample::Data const &data, std::size_t iDim, std::size_t iBin) +{ + std::vector const &edges = data.edges[iDim]; + if (data.uniform[iDim]) { + // The same as RooUniformBinning::binWidth() + return (edges.back() - edges.front()) / (edges.size() - 1); + } + return edges[iBin + 1] - edges[iBin]; +} + +std::size_t binInDim(RooHistFactorySample::Data const &data, std::size_t iDim, double x) +{ + std::vector const &edges = data.edges[iDim]; + const int nBins = edges.size() - 1; + if (data.uniform[iDim]) { + // The same as RooUniformBinning::binNumbers() + const double oneOverW = 1. / ((edges.back() - edges.front()) / nBins); + return x >= edges.back() ? nBins - 1 : std::min(nBins - 1, std::max(0, int((x - edges.front()) * oneOverW))); + } + return RooFit::Detail::MathFuncs::binNumber(x, 1.0, edges.data(), edges.size(), nBins - 1, 0); +} + +} // namespace + +/// Create a sample. +/// \param name The name of the RooFit object. +/// \param title The title of the RooFit object. +/// \param observables The observables, which need to be RooRealVars. Their binning defines the bins of the sample. +/// \param spec The bin contents and the modifiers of the sample. +RooHistFactorySample::RooHistFactorySample(const char *name, const char *title, RooArgList const &observables, + Spec const &spec) + : RooAbsReal(name, title), _obs("!obs", "observables", this), _params("!params", "parameters", this) +{ + auto data = std::make_shared(); + data->sampleName = spec.sampleName; + + std::size_t nBins = 1; + for (RooAbsArg *arg : observables) { + auto *obs = dynamic_cast(arg); + if (!obs) { + throw std::invalid_argument(std::string("RooHistFactorySample: the observable \"") + arg->GetName() + + "\" of sample \"" + spec.sampleName + "\" is not a RooRealVar"); + } + _obs.add(*obs); + RooAbsBinning const &binning = obs->getBinning(); + // Each call to RooBinning::array() reallocates the array. + double const *boundaries = binning.array(); + data->edges.emplace_back(boundaries, boundaries + binning.numBoundaries()); + data->uniform.push_back(binning.isUniform()); + // The array of a uniform binning is computed from the low bound and the bin width, so the high bound has to be + // set explicitly to find the bins exactly like RooUniformBinning. + if (binning.isUniform()) { + data->edges.back().back() = binning.highBound(); + } + nBins *= binning.numBins(); + } + + checkSize(spec.contents.size(), nBins, spec.sampleName, "the nominal histogram"); + data->contents = spec.contents; + if (!spec.errors.empty()) { + checkSize(spec.errors.size(), nBins, spec.sampleName, "the histogram of the statistical uncertainties"); + data->errors = spec.errors; + } + + for (NormFactor const &mod : spec.normFactors) { + data->normFactorNames.push_back(mod.name); + data->normFactorParams.push_back(checkedIndex(_params, mod.param, spec.sampleName, mod.name)); + } + + for (NormSys const &mod : spec.normSys) { + data->normSysNames.push_back(mod.name); + data->normSysParams.push_back(checkedIndex(_params, mod.param, spec.sampleName, mod.name)); + data->normSysLo.push_back(mod.lo); + data->normSysHi.push_back(mod.hi); + data->normSysCodes.push_back(mod.interpCode); + } + + for (HistoSys const &mod : spec.histoSys) { + checkSize(mod.lo.size(), nBins, spec.sampleName, "the low variation of \"" + mod.name + "\""); + checkSize(mod.hi.size(), nBins, spec.sampleName, "the high variation of \"" + mod.name + "\""); + data->histoSysNames.push_back(mod.name); + data->histoSysParams.push_back(checkedIndex(_params, mod.param, spec.sampleName, mod.name)); + data->histoSysCodes.push_back(mod.interpCode); + data->histoSysLo.insert(data->histoSysLo.end(), mod.lo.begin(), mod.lo.end()); + data->histoSysHi.insert(data->histoSysHi.end(), mod.hi.begin(), mod.hi.end()); + } + + for (BinFactor const &mod : spec.binFactors) { + checkSize(mod.params.size(), nBins, spec.sampleName, "the parameters of \"" + mod.name + "\""); + data->binFactorNames.push_back(mod.name); + data->binFactorTypes.push_back(static_cast(mod.type)); + for (RooAbsArg *param : mod.params) { + data->binFactorParams.push_back( + checkedIndex(_params, dynamic_cast(param), spec.sampleName, mod.name)); + } + } + + _data = std::move(data); + + // Validate the interpolation codes like the classes that implement them. + for (std::size_t i = 0; i < _data->normSysCodes.size(); ++i) { + if (_data->normSysCodes[i] < 0 || _data->normSysCodes[i] > 5) { + throw std::invalid_argument("RooHistFactorySample: unknown NormSys interpolation code " + + std::to_string(_data->normSysCodes[i]) + " for \"" + _data->normSysNames[i] + + "\""); + } + } + for (std::size_t i = 0; i < _data->histoSysCodes.size(); ++i) { + if (_data->histoSysCodes[i] < 0 || _data->histoSysCodes[i] > 6 || _data->histoSysCodes[i] == 3) { + throw std::invalid_argument("RooHistFactorySample: unknown HistoSys interpolation code " + + std::to_string(_data->histoSysCodes[i]) + " for \"" + _data->histoSysNames[i] + + "\""); + } + } +} + +RooHistFactorySample::RooHistFactorySample(const RooHistFactorySample &other, const char *name) + : RooAbsReal(other, name), + _obs("!obs", this, other._obs), + _params("!params", this, other._params), + _data(other._data), + _yieldsMode(other._yieldsMode) +{ +} + +/// The volume of a bin, i.e. the product of its widths in all observables. +double RooHistFactorySample::binVolume(std::size_t iBin) const +{ + Data const &data = *_data; + // Like RooDataHist, the bin volume is accumulated starting from the first observable. + double volume = 1.0; + std::size_t stride = data.numBins(); + for (std::size_t iDim = 0; iDim < data.edges.size(); ++iDim) { + stride /= data.edges[iDim].size() - 1; + volume *= binWidth(data, iDim, (iBin / stride) % (data.edges[iDim].size() - 1)); + } + return volume; +} + +/// The scale factor of the whole sample, i.e. the product of the NormSys factors and the NormFactors. +double RooHistFactorySample::scaleFactor(double const *params) const +{ + Data const &data = *_data; + return RooFit::Detail::MathFuncs::histFactoryScale( + params, data.normSysParams.size(), data.normSysParams.data(), data.normSysCodes.data(), data.normSysLo.data(), + data.normSysHi.data(), data.normFactorParams.size(), data.normFactorParams.data()); +} + +/// The value in a bin without the scale factor of the sample. +double RooHistFactorySample::binValue(double const *params, std::size_t iBin) const +{ + Data const &data = *_data; + return RooFit::Detail::MathFuncs::histFactoryBin( + params, iBin, data.numBins(), data.contents.data(), data.histoSysParams.size(), data.histoSysParams.data(), + data.histoSysCodes.data(), data.histoSysLo.data(), data.histoSysHi.data(), data.binFactorNames.size(), + data.binFactorParams.data()); +} + +template +void RooHistFactorySample::fillParams(ParamValue &¶mValue) const +{ + _paramVals.resize(_params.size()); + for (std::size_t i = 0; i < _params.size(); ++i) { + _paramVals[i] = paramValue(static_cast(_params[i])); + } +} + +/// The values of all bins at the current parameter values, either as densities or as yields. +void RooHistFactorySample::binValues(std::span out, bool density) const +{ + if (out.size() != numBins()) { + throw std::invalid_argument("RooHistFactorySample::binValues(): the output has the wrong size"); + } + fillParams([](RooAbsReal const ¶m) { return param.getVal(); }); + const double coef = scaleFactor(_paramVals.data()); + for (std::size_t i = 0; i < out.size(); ++i) { + double value = binValue(_paramVals.data(), i); + if (density) { + value *= 1. / binVolume(i); + } + out[i] = value * coef; + } +} + +std::size_t RooHistFactorySample::binIndex(std::span values) const +{ + Data const &data = *_data; + std::size_t index = 0; + for (std::size_t iDim = 0; iDim < data.edges.size(); ++iDim) { + index = index * (data.edges[iDim].size() - 1) + binInDim(data, iDim, values[iDim]); + } + return index; +} + +double RooHistFactorySample::evaluate() const +{ + fillParams([](RooAbsReal const ¶m) { return param.getVal(); }); + _obsVals.resize(_obs.size()); + for (std::size_t i = 0; i < _obs.size(); ++i) { + _obsVals[i] = static_cast(_obs[i]).getVal(); + } + const std::size_t iBin = binIndex(_obsVals); + double value = binValue(_paramVals.data(), iBin); + if (!_yieldsMode) { + value *= 1. / binVolume(iBin); + } + return value * scaleFactor(_paramVals.data()); +} + +void RooHistFactorySample::doEval(RooFit::EvalContext &ctx) const +{ + std::span output = ctx.output(); + fillParams([&](RooAbsReal const ¶m) { return ctx.at(¶m)[0]; }); + const double coef = scaleFactor(_paramVals.data()); + + std::vector> obsValues; + obsValues.reserve(_obs.size()); + for (RooAbsArg *obs : _obs) { + obsValues.push_back(ctx.at(obs)); + } + + // For many entries, all bins are computed only once. + const std::size_t nBins = numBins(); + const bool allBins = output.size() >= nBins; + if (allBins) { + _binVals.resize(nBins); + for (std::size_t i = 0; i < nBins; ++i) { + double value = binValue(_paramVals.data(), i); + if (!_yieldsMode) { + value *= 1. / binVolume(i); + } + _binVals[i] = value * coef; + } + } + + _obsVals.resize(_obs.size()); + for (std::size_t iEntry = 0; iEntry < output.size(); ++iEntry) { + for (std::size_t i = 0; i < _obs.size(); ++i) { + _obsVals[i] = obsValues[i].size() > 1 ? obsValues[i][iEntry] : obsValues[i][0]; + } + const std::size_t iBin = binIndex(_obsVals); + if (allBins) { + output[iEntry] = _binVals[iBin]; + continue; + } + double value = binValue(_paramVals.data(), iBin); + if (!_yieldsMode) { + value *= 1. / binVolume(iBin); + } + output[iEntry] = value * coef; + } +} + +int RooHistFactorySample::observableIndex(RooAbsArg const &obs) const +{ + for (std::size_t i = 0; i < _obs.size(); ++i) { + if (std::string(_obs[i].GetName()) == obs.GetName()) { + return i; + } + } + return -1; +} + +bool RooHistFactorySample::isBinnedDistribution(const RooArgSet &obs) const +{ + for (RooAbsArg *arg : obs) { + if (observableIndex(*arg) < 0) { + return false; + } + } + return true; +} + +std::list *RooHistFactorySample::binBoundaries(RooAbsRealLValue &obs, double xlo, double xhi) const +{ + const int iDim = observableIndex(obs); + if (iDim < 0) { + return nullptr; + } + // Same as RooHistFunc::binBoundaries() + const double delta = (xhi - xlo) * 1e-8; + auto hint = new std::list; + for (double boundary : _data->edges[iDim]) { + if (boundary > xlo - delta && boundary < xhi + delta) { + hint->push_back(boundary); + } + } + return hint; +} + +std::list *RooHistFactorySample::plotSamplingHint(RooAbsRealLValue &obs, double xlo, double xhi) const +{ + const int iDim = observableIndex(obs); + if (iDim < 0) { + return nullptr; + } + return RooCurve::plotSamplingHintForBinBoundaries(_data->edges[iDim], xlo, xhi); +} + +/// Integrals over any subset of the observables are analytical. The code is one plus the bit mask of the integrated +/// observables. +Int_t RooHistFactorySample::getAnalyticalIntegral(RooArgSet &allVars, RooArgSet &analVars, + const char * /*rangeName*/) const +{ + if (_forceNumInt || _yieldsMode) { + return 0; + } + Int_t code = 0; + for (std::size_t i = 0; i < _obs.size(); ++i) { + if (RooAbsArg *arg = allVars.find(_obs[i])) { + analVars.add(*arg); + code |= 1 << i; + } + } + return code == 0 ? 0 : code + 1; +} + +double RooHistFactorySample::analyticalIntegral(Int_t code, const char *rangeName) const +{ + Data const &data = *_data; + const std::vector weights = integralWeights(code, rangeName); + fillParams([](RooAbsReal const ¶m) { return param.getVal(); }); + const double integral = RooFit::Detail::MathFuncs::histFactoryIntegral( + _paramVals.data(), weights.data(), data.numBins(), data.contents.data(), data.histoSysParams.size(), + data.histoSysParams.data(), data.histoSysCodes.data(), data.histoSysLo.data(), data.histoSysHi.data(), + data.binFactorNames.size(), data.binFactorParams.data()); + return integral * scaleFactor(_paramVals.data()); +} + +/// The weights of the bins in an integral with the code from getAnalyticalIntegral(). For an integrated observable, it +/// is the fraction of the bin in the integration range. For an observable that is not integrated, only the bin of the +/// current value has a weight, which is the inverse of its width because the integral is a density in this observable. +std::vector RooHistFactorySample::integralWeights(Int_t code, const char *rangeName) const +{ + Data const &data = *_data; + const std::size_t nDims = data.edges.size(); + const int mask = code - 1; + + std::vector> weightsPerDim(nDims); + for (std::size_t iDim = 0; iDim < nDims; ++iDim) { + const std::size_t nBinsDim = data.edges[iDim].size() - 1; + auto const &obs = static_cast(_obs[iDim]); + std::vector &w = weightsPerDim[iDim]; + w.assign(nBinsDim, 0.0); + if (!(mask & (1 << iDim))) { + const std::size_t iBin = binInDim(data, iDim, obs.getVal()); + w[iBin] = 1. / binWidth(data, iDim, iBin); + continue; + } + const double lo = obs.getMin(rangeName); + const double hi = obs.getMax(rangeName); + for (std::size_t iBin = 0; iBin < nBinsDim; ++iBin) { + const double binLo = data.edges[iDim][iBin]; + const double binHi = data.edges[iDim][iBin + 1]; + if (binLo >= lo && binHi <= hi) { + w[iBin] = 1.0; + } else if (binHi > lo && binLo < hi) { + w[iBin] = (std::min(hi, binHi) - std::max(lo, binLo)) / binWidth(data, iDim, iBin); + } + } + } + + const std::size_t nBins = data.numBins(); + std::vector weights(nBins); + for (std::size_t iBin = 0; iBin < nBins; ++iBin) { + double weight = 1.0; + std::size_t stride = nBins; + for (std::size_t iDim = 0; iDim < nDims; ++iDim) { + const std::size_t nBinsDim = data.edges[iDim].size() - 1; + stride /= nBinsDim; + weight *= weightsPerDim[iDim][(iBin / stride) % nBinsDim]; + } + weights[iBin] = weight; + } + return weights; +} + +std::unique_ptr +RooHistFactorySample::compileForNormSet(RooArgSet const &normSet, RooFit::Detail::CompileContext &ctx) const +{ + // In the binned likelihood, the values of the pdf are directly the yields + // in the bins, like for a RooBinWidthFunction in that mode. + if (ctx.binnedLikelihoodMode()) { + auto newArg = std::make_unique(*this); + newArg->_yieldsMode = true; + ctx.markAsCompiled(*newArg); + ctx.compileServers(*newArg, {}); + ctx.setBinWidthFuncFlag(true); + return newArg; + } + return RooAbsReal::compileForNormSet(normSet, ctx); +} + +void RooHistFactorySample::setInterpCode(std::vector Data::*codes, std::vector Data::*params, + RooAbsReal const ¶m, int code, int maxCode, const char *what) +{ + // By name, so that the parameter can also come from another copy of the model. + const int iParam = _params.index(param.GetName()); + auto data = std::make_shared(*_data); + bool found = false; + for (std::size_t i = 0; i < (data.get()->*params).size(); ++i) { + if ((data.get()->*params)[i] == iParam) { + RooStats::HistFactory::Detail::setInterpolationCode(*this, what, param, data.get()->*codes, i, code, maxCode); + found = true; + } + } + if (!found) { + coutE(InputArguments) << what << "::setInterpCode ERROR: " << param.GetName() << " is not a parameter of a " + << what << " modifier of sample \"" << sampleName() << "\"" << std::endl; + return; + } + _data = std::move(data); + setValueDirty(); +} + +/// Set the interpolation code of the HistoSys modifiers with the given parameter. The codes are the ones of +/// PiecewiseInterpolation. +void RooHistFactorySample::setHistoSysInterpCode(RooAbsReal const ¶m, int code) +{ + setInterpCode(&Data::histoSysCodes, &Data::histoSysParams, param, code, 6, "HistoSys"); +} + +/// Set the interpolation code of the NormSys modifiers with the given parameter. The codes are the ones of +/// FlexibleInterpVar. +void RooHistFactorySample::setNormSysInterpCode(RooAbsReal const ¶m, int code) +{ + setInterpCode(&Data::normSysCodes, &Data::normSysParams, param, code, 5, "NormSys"); +} + +void RooHistFactorySample::printMetaArgs(std::ostream &os) const +{ + os << "sample=" << sampleName() << " bins=" << numBins(); + auto printNames = [&](const char *label, std::vector const &names) { + if (names.empty()) { + return; + } + os << " " << label << "=("; + for (std::size_t i = 0; i < names.size(); ++i) { + os << (i > 0 ? "," : "") << names[i]; + } + os << ")"; + }; + printNames("normfactors", _data->normFactorNames); + printNames("normsys", _data->normSysNames); + printNames("histosys", _data->histoSysNames); + printNames("binfactors", _data->binFactorNames); + os << " "; +} + +void RooHistFactorySample::Streamer(TBuffer &buffer) +{ + static TClass *dataClass = TClass::GetClass(); + if (buffer.IsReading()) { + buffer.ReadClassBuffer(RooHistFactorySample::Class(), this); + auto data = std::make_shared(); + buffer.StreamObject(data.get(), dataClass); + _data = std::move(data); + } else { + buffer.WriteClassBuffer(RooHistFactorySample::Class(), this); + buffer.StreamObject(const_cast(_data.get()), dataClass); + } +} diff --git a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h index 7cc6ab7086791..f5237441f9f8b 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h +++ b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h @@ -371,6 +371,76 @@ double flexibleInterp(unsigned int code, ParamsArray params, unsigned int n, Dou return doCutoff && total <= 0 ? TMath::Limits::Min() : total; } +/// The scale factor of a sample of a HistFactory model (RooHistFactorySample): the product of the NormSys factors, +/// interpolated like in FlexibleInterpVar, and of the NormFactors. +template +double +histFactoryScale(ParamsArray params, unsigned int nNormSys, int const *normSysParams, int const *normSysCodes, + DoubleArray normSysLo, DoubleArray normSysHi, unsigned int nNormFactors, int const *normFactorParams) +{ + double coef = 1.0; + if (nNormSys > 0) { + double total = 1.0; + for (unsigned int i = 0; i < nNormSys; ++i) { + // Code 4 of FlexibleInterpVar is code 5 of PiecewiseInterpolation. + const unsigned int code = normSysCodes[i] == 4 ? 5 : normSysCodes[i]; + total += flexibleInterpSingle(code, normSysLo[i], normSysHi[i], 1.0, 1.0, params[normSysParams[i]], total); + } + if (total <= 0) { + total = TMath::Limits::Min(); + } + coef *= total; + } + for (unsigned int i = 0; i < nNormFactors; ++i) { + coef *= params[normFactorParams[i]]; + } + return coef; +} + +/// The content of a bin of a sample of a HistFactory model (RooHistFactorySample) without the scale factor: the +/// nominal content with the HistoSys variations, interpolated like in PiecewiseInterpolation and clamped at zero, +/// times the parameters of the bin-wise modifiers. +template +double +histFactoryBin(ParamsArray params, unsigned int iBin, unsigned int nBins, DoubleArray nominal, unsigned int nHistoSys, + int const *histoSysParams, int const *histoSysCodes, DoubleArray histoSysLo, DoubleArray histoSysHi, + unsigned int nBinFactors, int const *binFactorParams) +{ + const double nom = nominal[iBin]; + double value = nom; + if (nHistoSys > 0) { + double sum = nom; + for (unsigned int i = 0; i < nHistoSys; ++i) { + sum += flexibleInterpSingle(histoSysCodes[i], histoSysLo[i * nBins + iBin], histoSysHi[i * nBins + iBin], 1.0, + nom, params[histoSysParams[i]], sum); + } + value = sum < 0 ? 0 : sum; + } + for (unsigned int i = 0; i < nBinFactors; ++i) { + value *= params[binFactorParams[i * nBins + iBin]]; + } + return value; +} + +/// The weighted sum of the bins of a sample of a HistFactory model (RooHistFactorySample) without the scale factor. +/// With the fractions of the bins in the integration range as weights, this is the integral of the sample. +template +double +histFactoryIntegral(ParamsArray params, DoubleArray weights, unsigned int nBins, DoubleArray nominal, + unsigned int nHistoSys, int const *histoSysParams, int const *histoSysCodes, DoubleArray histoSysLo, + DoubleArray histoSysHi, unsigned int nBinFactors, int const *binFactorParams) +{ + double out = 0.0; + for (unsigned int i = 0; i < nBins; ++i) { + if (weights[i] != 0.0) { + out += histFactoryBin(params, i, nBins, nominal, nHistoSys, histoSysParams, histoSysCodes, histoSysLo, + histoSysHi, nBinFactors, binFactorParams) * + weights[i]; + } + } + return out; +} + inline double landau(double x, double mu, double sigma) { if (sigma <= 0.) From fce3968384c985c133d2f597c84ea7fc5f22c528 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 16 Nov 2026 15:36:24 +0000 Subject: [PATCH 2/5] [RF][HF] Build HistFactory channels as RooHistFactoryPdf by default HistFactory and the HS3 importer now create one RooHistFactorySample per sample instead of several RooFit objects per histogram. The previous structure is still available with HistoToWorkspaceFactoryFast::Configuration::useHistFactoryPdf, `hist2workspace -legacy_model_structure`, and RooJSONFactoryWSTool::config().useHistFactoryPdf. The HS3 exporter writes RooHistFactoryPdf directly, keeping the names of the modifiers. The tests that inspect the RooFit objects of the models are adapted, and the ones about exporting the previous structure use it explicitly. Assisted-by: claude-code:claude-opus-5-5 --- README/ReleaseNotes/v644/index.md | 32 ++ .../HistFactory/HistoToWorkspaceFactoryFast.h | 3 + .../histfactory/src/HistFactoryModelUtils.cxx | 7 +- .../histfactory/src/HistFactoryNavigation.cxx | 8 +- .../src/HistoToWorkspaceFactoryFast.cxx | 408 ++++++++++++++--- roofit/histfactory/src/hist2workspace.cxx | 15 +- roofit/histfactory/test/CMakeLists.txt | 5 + roofit/histfactory/test/testHistFactory.cxx | 55 ++- .../test/testRooHistFactoryPdf.cxx | 306 +++++++++++++ .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 4 + roofit/hs3/src/JSONFactories_HistFactory.cxx | 420 ++++++++++++++++-- roofit/hs3/test/HS3TestUtils.h | 4 + roofit/hs3/test/testRooFitHS3.cxx | 251 ++++++++--- roofit/man/man1/hist2workspace.1 | 4 +- 14 files changed, 1325 insertions(+), 197 deletions(-) create mode 100644 roofit/histfactory/test/testRooHistFactoryPdf.cxx diff --git a/README/ReleaseNotes/v644/index.md b/README/ReleaseNotes/v644/index.md index dde65790e5f56..a5f7498c21bf3 100644 --- a/README/ReleaseNotes/v644/index.md +++ b/README/ReleaseNotes/v644/index.md @@ -52,6 +52,38 @@ RenderCore, the WebGL engine of the REve client, is fetched at build time as the ## RooFit +### HistFactory models with fewer RooFit objects + +HistFactory now builds each channel as a `RooHistFactoryPdf`, which corresponds +to the `histfactory_dist` of the HS3 standard. Each sample of a channel is a +single `RooHistFactorySample` object, which holds the nominal histogram and all +its modifiers (NormFactor, OverallSys, HistoSys, StatError, ShapeSys and +ShapeFactor). Before, HistFactory created several RooFit objects for each +histogram (`RooHistFunc`, `RooDataHist`, `PiecewiseInterpolation`, +`FlexibleInterpVar`, `ParamHistFunc` and `RooProduct`). For models with many +systematic variations, this reduces the size of the workspaces in memory and on +disk. The histogram data of a sample is shared between its copies, so cloning +the model, e.g. when creating the likelihood, doesn't copy the histograms. + +The models have the same parameters, constraint terms, observables and datasets +as before, and they give the same values up to floating point rounding. The +binned likelihood optimization, +the code generation backend and the plotting of single samples with +`RooFit::Components("__shapes")` work as before. Code that +navigates the internal structure of HistFactory models, e.g. to retrieve the +`RooHistFunc` of a sample, has to be adapted. The modifiers of a sample are +available via `RooHistFactorySample::data()`. + +The HS3 importer also creates `RooHistFactoryPdf` objects for the +`histfactory_dist` distributions, and the exporter writes them with the names +of their modifiers. + +To get the previous model structure, set the `useHistFactoryPdf` member of the +`RooStats::HistFactory::HistoToWorkspaceFactoryFast::Configuration` to `false`, +use the `-legacy_model_structure` command line argument of `hist2workspace`, or +set `RooJSONFactoryWSTool::config().useHistFactoryPdf = false` for the HS3 +importer. Workspaces with the previous structure can still be read. + ### Removal of the legacy evaluation backend The `legacy` evaluation backend for likelihood and chi-square fits is removed. diff --git a/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h b/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h index a9ca37eaf6bed..807b65e6054fe 100644 --- a/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h +++ b/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h @@ -53,6 +53,9 @@ class HistoToWorkspaceFactoryFast : public TObject { bool createPerRegionWorkspaces = true; /// \brief Control whether errors on the data histograms are stored in the workspace (default `false`). bool storeDataError = false; + /// \brief Build each channel as a RooHistFactoryPdf of RooHistFactorySample objects (default `true`). If + /// `false`, the channels are built as before ROOT 6.44, with several RooFit objects for each histogram. + bool useHistFactoryPdf = true; }; HistoToWorkspaceFactoryFast() {} diff --git a/roofit/histfactory/src/HistFactoryModelUtils.cxx b/roofit/histfactory/src/HistFactoryModelUtils.cxx index 729d71a3dba89..d9fa29629b502 100644 --- a/roofit/histfactory/src/HistFactoryModelUtils.cxx +++ b/roofit/histfactory/src/HistFactoryModelUtils.cxx @@ -7,6 +7,7 @@ #include #include "RooStats/HistFactory/ParamHistFunc.h" +#include "RooRealSumPdf.h" #include "RooAbsPdf.h" #include "RooArgSet.h" #include "RooArgList.h" @@ -41,10 +42,10 @@ namespace HistFactory{ bool FoundSumPdf=false; std::unique_ptr components{sim_channel->getComponents()}; for (auto *sum_pdf_arg : *components) { - std::string NodeClassName = sum_pdf_arg->ClassName(); - if( NodeClassName == std::string("RooRealSumPdf") ) { + // Also finds a RooHistFactoryPdf, which is a RooRealSumPdf of its samples + if (auto *sumPdf = dynamic_cast(sum_pdf_arg)) { FoundSumPdf=true; - sum_pdf = static_cast(sum_pdf_arg); + sum_pdf = sumPdf; break; } } diff --git a/roofit/histfactory/src/HistFactoryNavigation.cxx b/roofit/histfactory/src/HistFactoryNavigation.cxx index 7ae284bf19932..27ccdf0de4b87 100644 --- a/roofit/histfactory/src/HistFactoryNavigation.cxx +++ b/roofit/histfactory/src/HistFactoryNavigation.cxx @@ -788,10 +788,10 @@ namespace RooStats { // the "unconstrained" pdf node for that channel std::unique_ptr comps{pdf->getComponents()}; for (auto *arg : *comps) { - std::string ClassName = arg->ClassName(); - if( ClassName == "RooRealSumPdf" ) { - fChannelSumNodeMap[ChannelName] = static_cast (arg); - break; + // Also finds a RooHistFactoryPdf, which is a RooRealSumPdf of its samples + if (auto *sumPdf = dynamic_cast(arg)) { + fChannelSumNodeMap[ChannelName] = sumPdf; + break; } } } diff --git a/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx b/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx index 14e284fac6453..e42aad6d00101 100644 --- a/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx +++ b/roofit/histfactory/src/HistoToWorkspaceFactoryFast.cxx @@ -48,6 +48,8 @@ #include "RooStats/RooStatsUtils.h" #include "RooStats/HistFactory/PiecewiseInterpolation.h" #include "RooStats/HistFactory/ParamHistFunc.h" +#include "RooStats/HistFactory/RooHistFactoryPdf.h" +#include "RooStats/HistFactory/RooHistFactorySample.h" #include "RooStats/AsymptoticCalculator.h" #include "HFMsgService.h" @@ -63,6 +65,7 @@ #include "RooStats/HistFactory/HistFactoryException.h" #include +#include #include #include #include @@ -88,6 +91,49 @@ std::vector histToVector(TH1 const &hist) return out; } +namespace { + +/// The bin contents (or errors) of a histogram in the order of the bins of RooDataHist and HS3: the last axis varies +/// fastest. +std::vector histToBinOrder(TH1 const &hist, bool errors = false) +{ + const int nx = hist.GetNbinsX(); + const int ny = hist.GetDimension() > 1 ? hist.GetNbinsY() : 1; + const int nz = hist.GetDimension() > 2 ? hist.GetNbinsZ() : 1; + std::vector out; + out.reserve(nx * ny * nz); + for (int ix = 1; ix <= nx; ++ix) { + for (int iy = 1; iy <= ny; ++iy) { + for (int iz = 1; iz <= nz; ++iz) { + const int bin = hist.GetBin(ix, iy, iz); + out.push_back(errors ? hist.GetBinError(bin) : hist.GetBinContent(bin)); + } + } + } + return out; +} + +/// ParamHistFunc::createParamSet() lists the parameters with the first axis varying fastest, like the bins of a TH1. +/// This reorders them like the bins of RooDataHist and HS3, where the last axis varies fastest. +RooArgList paramsToBinOrder(RooArgList const ¶ms, RooArgList const &observables) +{ + std::array n{1, 1, 1}; + for (std::size_t i = 0; i < observables.size(); ++i) { + n[i] = static_cast(observables[i]).numBins(); + } + RooArgList out; + for (int ix = 0; ix < n[0]; ++ix) { + for (int iy = 0; iy < n[1]; ++iy) { + for (int iz = 0; iz < n[2]; ++iz) { + out.add(params[ix + n[0] * (iy + n[1] * iz)]); + } + } + } + return out; +} + +} // namespace + // use this order for safety on library loading using namespace RooStats; using std::string, std::vector; @@ -578,20 +624,23 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo return normFactor; } - void HistoToWorkspaceFactoryFast::AddConstraintTerms(RooWorkspace& proto, Measurement & meas, string prefix, - string interpName, - std::vector& systList, - vector& constraintTermNames, - vector& totSystTermNames) { + namespace { - // add variables for all the relative overall uncertainties we expect - totSystTermNames.push_back(prefix); + /// A parameter of a NormSys modifier (OverallSys in HistFactory) with its variations. + struct OverallSysTerm { + std::string name; + RooAbsReal *param = nullptr; + double low = 1.0; + double high = 1.0; + }; - RooArgSet params(prefix.c_str()); - vector lowVec; - vector highVec; + /// Create the parameters of the OverallSys modifiers of a sample and their constraint terms. + std::vector makeOverallSysTerms(RooWorkspace& proto, Measurement & meas, string const& prefix, + std::vector& systList, + vector& constraintTermNames) { + + std::vector terms; - std::map::iterator itconstr; for(unsigned int i = 0; i < systList.size(); ++i) { OverallSys& sys = systList.at(i); @@ -600,14 +649,14 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // case of no systematic (is it possible) if (meas.GetNoSyst().count(sys.GetName()) > 0 ) { - cxcoutI(HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - skip systematic " << sys.GetName() << std::endl; + oocxcoutI(nullptr, HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - skip systematic " << sys.GetName() << std::endl; continue; } // case systematic is a gamma constraint if (meas.GetGammaSyst().count(sys.GetName()) > 0 ) { double relerr = meas.GetGammaSyst().find(sys.GetName() )->second; if (relerr <= 0) { - cxcoutI(HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - zero uncertainty assigned - skip systematic " << sys.GetName() << std::endl; + oocxcoutI(nullptr, HistFactory) << "HistoToWorkspaceFast::AddConstraintTerm - zero uncertainty assigned - skip systematic " << sys.GetName() << std::endl; continue; } const double tauVal = 1./(relerr*relerr); @@ -634,8 +683,8 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo const_cast(proto.set("globalObservables"))->add(yvar); // add alphaOfBeta in the list of params to interpolate - params.add(alphaOfBeta); - cxcoutI(HistFactory) << "Added a gamma constraint for " << name << std::endl; + terms.push_back({sys.GetName(), &alphaOfBeta, sys.GetLow(), sys.GetHigh()}); + oocxcoutI(nullptr, HistFactory) << "Added a gamma constraint for " << name << std::endl; } else { @@ -647,7 +696,7 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // check if exists a log-normal constraint if (meas.GetLogNormSyst().count(sys.GetName()) == 0 && meas.GetGammaSyst().count(sys.GetName()) == 0 ) { // just add the alpha for the parameters of the FlexibleInterpVar function - params.add(alpha); + terms.push_back({sys.GetName(), &alpha, sys.GetLow(), sys.GetHigh()}); } // case systematic is a log-normal constraint if (meas.GetLogNormSyst().count(sys.GetName()) > 0 ) { @@ -659,20 +708,206 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo RooArgList{emplace(proto, "tau_" + sys.GetName(), 1. / relerr), emplace(proto, "kappa_" + sys.GetName(), 1. + relerr), alpha}); - cxcoutI(HistFactory) << "Added a log-normal constraint for " << name << std::endl; + oocxcoutI(nullptr, HistFactory) << "Added a log-normal constraint for " << name << std::endl; if (RooMsgService::instance().isActive(nullptr, RooFit::HistFactory, RooFit::DEBUG)) { alphaOfBeta.Print("t"); } - params.add(alphaOfBeta); + terms.push_back({sys.GetName(), &alphaOfBeta, sys.GetLow(), sys.GetHigh()}); } } - // add low/high vectors - lowVec.push_back(sys.GetLow()); - highVec.push_back(sys.GetHigh()); } // end sys loop + return terms; + } + + /// Create the parameters of the NormFactors of a sample, and return their names. + std::vector makeNormFactorParams(RooWorkspace& proto, std::string const& channel, Sample& sample, + bool doRatio) { + std::vector names; + for (NormFactor const& norm : sample.GetNormFactorList()) { + string varname = norm.GetName(); + if (doRatio) { + varname += "_" + channel; + } + // The NormFactor may already exist, e.g. because it was created as a function expression during preprocessing. + if (proto.obj(varname) == nullptr) { + oocxcoutI(nullptr, HistFactory) << "making normFactor: " << norm.GetName() << std::endl; + emplace(proto, varname, norm.GetVal(), norm.GetLow(), norm.GetHigh()); + proto.var(varname)->setError(0); // ensure factor is assigned an initial error, even if its zero + } + if (std::find(names.begin(), names.end(), varname) != names.end()) { + oocxcoutI(nullptr, HistFactory) << " is duplicated for , but only one factor will be included." << std::endl; + continue; + } + names.push_back(varname); + } + return names; + } + + /// Collect everything about a sample for the RooHistFactorySample, creating the parameters and constraint terms of + /// its modifiers like for the models before ROOT 6.44. + RooHistFactorySample::Spec makeSampleSpec(RooWorkspace &proto, Measurement &measurement, + std::string const &channelName, Sample &sample, + RooArgList const &observables, bool doRatio, + std::vector &constraintTermNames, RooArgList &statGammas, + std::map &shapeFactorGammas, + std::map &shapeSysGammas) + { + using BinFactorType = RooHistFactorySample::BinFactorType; + + RooHistFactorySample::Spec spec; + spec.sampleName = sample.GetName(); + + const TH1 *nominal = sample.GetHisto(); + oocxcoutI(nullptr, HistFactory) << "processing hist " << nominal->GetName() << std::endl; + spec.contents = histToBinOrder(*nominal); + + // Same order of factors as in the models before ROOT 6.44: OverallSys, NormFactors, Lumi + for (OverallSysTerm const &term : makeOverallSysTerms(proto, measurement, "alpha_", sample.GetOverallSysList(), + constraintTermNames)) { + // Interpolation code 5 of FlexibleInterpVar: polynomial interpolation and exponential extrapolation + spec.normSys.push_back({term.name, term.param, term.low, term.high, 5}); + } + + for (std::string const &name : makeNormFactorParams(proto, channelName, sample, doRatio)) { + auto *param = dynamic_cast(proto.arg(name)); + if (!param) { + throw hf_exc("The NormFactor \"" + name + "\" of sample \"" + sample.GetName() + "\" is not a real value."); + } + spec.normFactors.push_back({name, param}); + } + + if (!sample.GetHistoSysList().empty()) { + RooArgList interpParams = makeInterpolationParameters(sample.GetHistoSysList(), proto); + for (std::size_t i = 0; i < interpParams.size(); ++i) { + HistoSys const &histoSys = sample.GetHistoSysList()[i]; + const bool isUniform = measurement.GetUniformSyst().count(histoSys.GetName()) > 0; + makeGaussianConstraint(interpParams[i], proto, isUniform, constraintTermNames); + // Interpolation code 4 of PiecewiseInterpolation: polynomial interpolation and linear extrapolation + spec.histoSys.push_back({histoSys.GetName(), &static_cast(interpParams[i]), + histToBinOrder(*histoSys.GetHistoLow()), histToBinOrder(*histoSys.GetHistoHigh()), + 4}); + } + } + + if (sample.GetStatError().GetActivate()) { + if (statGammas.empty()) { + statGammas.add(ParamHistFunc::createParamSet(proto, "gamma_stat_" + channelName, observables, + defaultGammaMin, defaultStatErrorGammaMax)); + } + spec.binFactors.push_back({"staterror", BinFactorType::StatError, paramsToBinOrder(statGammas, observables)}); + } + + for (ShapeFactor &shapeFactor : sample.GetShapeFactorList()) { + oocxcoutI(nullptr, HistFactory) << "Sample: " << sample.GetName() << " in channel: " << channelName + << " to be include a ShapeFactor." << std::endl; + auto found = shapeFactorGammas.find(shapeFactor.GetName()); + if (found == shapeFactorGammas.end()) { + RooArgList gammas = ParamHistFunc::createParamSet(proto, "gamma_" + shapeFactor.GetName(), observables); + for (auto *comp : gammas) { + // If the gamma is subject to a preprocess function, it is a RooAbsReal and + // we don't need to set the initial value. + if (auto var = dynamic_cast(comp)) { + var->setVal(shapeFactor.GetVal()); + var->setMin(shapeFactor.GetLow()); + var->setMax(shapeFactor.GetHigh()); + } + } + // Set an initial shape, if requested + if (TH1 const *initialShape = shapeFactor.GetInitialShape()) { + oocxcoutI(nullptr, HistFactory) << "Setting Shape Factor: " << shapeFactor.GetName() + << " to have initial shape from hist: " << initialShape->GetName() << std::endl; + std::vector values = histToVector(*initialShape); + if (values.size() != gammas.size()) { + throw hf_exc("The initial shape of the ShapeFactor \"" + shapeFactor.GetName() + + "\" has the wrong number of bins."); + } + for (std::size_t i = 0; i < gammas.size(); ++i) { + auto *var = dynamic_cast(&gammas[i]); + if (!var) { + throw hf_exc("Cannot set the initial shape of ShapeFactor \"" + shapeFactor.GetName() + + "\": a parameter is not a RooRealVar."); + } + var->setVal(values[i]); + } + } + // Set the variables constant, if requested + if (shapeFactor.IsConstant()) { + oocxcoutI(nullptr, HistFactory) << "Setting Shape Factor: " << shapeFactor.GetName() << " to be constant" << std::endl; + for (auto *comp : gammas) { + if (auto var = dynamic_cast(comp)) { + var->setConstant(true); + } + } + } + found = shapeFactorGammas.emplace(shapeFactor.GetName(), gammas).first; + } + spec.binFactors.push_back( + {shapeFactor.GetName(), BinFactorType::ShapeFactor, paramsToBinOrder(found->second, observables)}); + } + + for (ShapeSys &shapeSys : sample.GetShapeSysList()) { + oocxcoutI(nullptr, HistFactory) << "Sample: " << sample.GetName() << " in channel: " << channelName + << " to include a ShapeSys." << std::endl; + auto found = shapeSysGammas.find(shapeSys.GetName()); + if (found == shapeSysGammas.end()) { + RooArgList gammas = ParamHistFunc::createParamSet(proto, "gamma_" + shapeSys.GetName(), observables, + defaultGammaMin, defaultShapeSysGammaMax); + found = shapeSysGammas.emplace(shapeSys.GetName(), gammas).first; + } + // The constraint terms, from the relative errors of this sample + auto shapeConstraintsInfo = createGammaConstraints(found->second, histToVector(*shapeSys.GetErrorHist()), + minShapeUncertainty, shapeSys.GetConstraintType()); + for (auto const &term : shapeConstraintsInfo.constraints) { + proto.import(*term, RooFit::RecycleConflictNodes()); + constraintTermNames.emplace_back(term->GetName()); + } + // Add the "observed" value to the list of global observables: + RooArgSet *globalSet = const_cast(proto.set("globalObservables")); + for (RooAbsArg *glob : shapeConstraintsInfo.globalObservables) { + globalSet->add(*proto.var(glob->GetName())); + } + spec.binFactors.push_back( + {shapeSys.GetName(), BinFactorType::ShapeSys, paramsToBinOrder(found->second, observables)}); + } + + // The luminosity, as for the models before ROOT 6.44 + RooAbsArg *lumi = proto.arg("Lumi"); + if (!sample.GetNormalizeByTheory()) { + if (!lumi) { + lumi = &emplace(proto, "Lumi", measurement.GetLumi()); + } else { + static_cast(lumi)->setVal(measurement.GetLumi()); + } + } + spec.normFactors.push_back({"Lumi", &static_cast(*lumi)}); + + return spec; + } + + } // namespace + + void HistoToWorkspaceFactoryFast::AddConstraintTerms(RooWorkspace& proto, Measurement & meas, string prefix, + string interpName, + std::vector& systList, + vector& constraintTermNames, + vector& totSystTermNames) { + + // add variables for all the relative overall uncertainties we expect + totSystTermNames.push_back(prefix); + + RooArgSet params(prefix.c_str()); + vector lowVec; + vector highVec; + for (OverallSysTerm const& term : makeOverallSysTerms(proto, meas, prefix, systList, constraintTermNames)) { + params.add(*term.param); + lowVec.push_back(term.low); + highVec.push_back(term.high); + } + if(!systList.empty()){ // this is epsilon(alpha_j), a piece-wise linear interpolation // LinInterpVar interp( (interpName).c_str(), "", params, 1., lowVec, highVec); @@ -880,6 +1115,48 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo std::vector>> statHistPairs; // const std::string statFuncName = "mc_stat_" + channel_name; + // The absolute statistical uncertainties of a sample. + auto makeStatErrorHist = [&](Sample &sample) { + const TH1 *nominal = sample.GetHisto(); + string UncertName = sample.GetName() + "_" + channel_name + "_StatAbsolUncert"; + std::unique_ptr statErrorHist; + + if( sample.GetStatError().GetErrorHist() == nullptr ) { + // Make the absolute stat error + cxcoutI(HistFactory) << "Making Statistical Uncertainty Hist for " + << " Channel: " << channel_name + << " Sample: " << sample.GetName() + << std::endl; + statErrorHist.reset(MakeAbsolUncertaintyHist( UncertName, nominal)); + } else { + // clone the error histograms because in case the sample has not error hist + // it is created in MakeAbsolUncertainty + // we need later to clean statErrorHist + statErrorHist.reset(static_cast(sample.GetStatError().GetErrorHist()->Clone())); + // We assume the (relative) error is provided. + // We must turn it into an absolute error + // using the nominal histogram + cxcoutI(HistFactory) << "Using external histogram for Stat Errors for " + << "\tChannel: " << channel_name + << "\tSample: " << sample.GetName() + << "\tError Histogram: " << statErrorHist->GetName() << std::endl; + // Multiply the relative stat uncertainty by the + // nominal to get the overall stat uncertainty + statErrorHist->Multiply( nominal ); + statErrorHist->SetName( UncertName.c_str() ); + } + return statErrorHist; + }; + + // For the RooHistFactoryPdf: the samples, and the parameters of the + // modifiers that act on each bin, which samples can share. The parameters + // are in the order of ParamHistFunc::createParamSet(). + std::vector sampleSpecs; + std::vector sampleTitles; + RooArgList statGammas; + std::map shapeFactorGammas; + std::map shapeSysGammas; + ///////////////////////////// // shared parameters // this is ratio of lumi to nominal lumi. We will include relative uncertainty in model @@ -903,6 +1180,26 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // and terms that constrain floating to expectation via uncertainties // GHL: Loop over samples instead, which doesn't contain the data for (Sample& sample : channel.GetSamples()) { + if (fCfg.useHistFactoryPdf) { + RooArgList observables = createObservables(sample.GetHisto(), proto); + if (static_cast(sample.GetHisto()->GetDimension()) != fObsNameVec.size()) { + throw hf_exc("The histogram of sample \"" + sample.GetName() + "\" in channel \"" + channel_name + + "\" has another dimension than the other histograms of the channel."); + } + sampleSpecs.push_back(makeSampleSpec(proto, measurement, channel_name, sample, observables, doRatio, + constraintTermNames, statGammas, shapeFactorGammas, shapeSysGammas)); + RooHistFactorySample::Spec &spec = sampleSpecs.back(); + const TH1 *nominal = sample.GetHisto(); + sampleTitles.emplace_back(strlen(nominal->GetTitle()) > 0 ? nominal->GetTitle() : sample.GetName()); + if (sample.GetStatError().GetActivate()) { + cxcoutI(HistFactory) << "Sample: " << sample.GetName() << " to be included in Stat Error " + << "for channel " << channel_name << std::endl; + statHistPairs.emplace_back(nominal, makeStatErrorHist(sample)); + spec.errors = histToBinOrder(*statHistPairs.back().second); + } + continue; + } + string overallSystName = sample.GetName() + "_" + channel_name + "_epsilon"; string systSourcePrefix = "alpha_"; @@ -989,37 +1286,9 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo << "for channel " << channel_name << std::endl; - string UncertName = sample.GetName() + "_" + channel_name + "_StatAbsolUncert"; - std::unique_ptr statErrorHist; - - if( sample.GetStatError().GetErrorHist() == nullptr ) { - // Make the absolute stat error - cxcoutI(HistFactory) << "Making Statistical Uncertainty Hist for " - << " Channel: " << channel_name - << " Sample: " << sample.GetName() - << std::endl; - statErrorHist.reset(MakeAbsolUncertaintyHist( UncertName, nominal)); - } else { - // clone the error histograms because in case the sample has not error hist - // it is created in MakeAbsolUncertainty - // we need later to clean statErrorHist - statErrorHist.reset(static_cast(sample.GetStatError().GetErrorHist()->Clone())); - // We assume the (relative) error is provided. - // We must turn it into an absolute error - // using the nominal histogram - cxcoutI(HistFactory) << "Using external histogram for Stat Errors for " - << "\tChannel: " << channel_name - << "\tSample: " << sample.GetName() - << "\tError Histogram: " << statErrorHist->GetName() << std::endl; - // Multiply the relative stat uncertainty by the - // nominal to get the overall stat uncertainty - statErrorHist->Multiply( nominal ); - statErrorHist->SetName( UncertName.c_str() ); - } - // Save the nominal and error hists // for the building of constraint terms - statHistPairs.emplace_back(nominal, std::move(statErrorHist)); + statHistPairs.emplace_back(nominal, makeStatErrorHist(sample)); // To do the 'conservative' version, we would need to do some // intervention here. We would probably need to create a different @@ -1278,11 +1547,11 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo // Using this TH1* of fractinal stat errors, // create a set of constraint terms: - auto chanStatUncertFunc = static_cast(proto.function( statFuncName )); - cxcoutI(HistFactory) << "About to create Constraint Terms from: " - << chanStatUncertFunc->GetName() - << " params: " << chanStatUncertFunc->paramList() - << std::endl; + RooArgList const &statParams = fCfg.useHistFactoryPdf + ? statGammas + : static_cast(proto.function(statFuncName))->paramList(); + cxcoutI(HistFactory) << "About to create Constraint Terms from: " << statFuncName << " params: " << statParams + << std::endl; // Get the constraint type and the // rel error threshold from the (last) @@ -1300,7 +1569,7 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo double statRelErrorThreshold = channel.GetStatErrorConfig().GetRelErrorThreshold(); auto statConstraintsInfo = createGammaConstraints( - chanStatUncertFunc->paramList(), histToVector(*fracStatError), + statParams, histToVector(*fracStatError), statRelErrorThreshold, statConstraintType); for (auto const& term : statConstraintsInfo.constraints) { @@ -1318,8 +1587,31 @@ RooArgList HistoToWorkspaceFactoryFast::createObservables(const TH1 *hist, RooWo /////////////////////////////////// // for ith bin calculate totN_i = lumi * sum_j expected_j * syst_j - MakeTotalExpected(proto, channel_name+"_model", - sampleScaleFactors, allSampleHistFuncs); + if (fCfg.useHistFactoryPdf) { + const std::string totName = channel_name + "_model"; + RooArgList observables; + for (std::string const &name : fObsNameVec) { + observables.add(*proto.var(name)); + } + RooArgList samples; + for (std::size_t i = 0; i < sampleSpecs.size(); ++i) { + const std::string name = sampleSpecs[i].sampleName + "_" + channel_name + "_shapes"; + RooHistFactorySample sampleFunc{name.c_str(), sampleTitles[i].c_str(), observables, sampleSpecs[i]}; + proto.import(sampleFunc, RooFit::RecycleConflictNodes()); + samples.add(*proto.function(name)); + } + RooHistFactoryPdf tot{totName.c_str(), totName.c_str(), samples}; + // for mixed generation in RooSimultaneous + tot.setAttribute("GenerateBinned"); + // Enable the binned likelihood optimization + if (fCfg.binnedFitOptimization) { + tot.setAttribute("BinnedLikelihood"); + } + proto.import(tot, RooFit::RecycleConflictNodes()); + } else { + MakeTotalExpected(proto, channel_name+"_model", + sampleScaleFactors, allSampleHistFuncs); + } likelihoodTermNames.push_back(channel_name+"_model"); ////////////////////////////////////// diff --git a/roofit/histfactory/src/hist2workspace.cxx b/roofit/histfactory/src/hist2workspace.cxx index c0c1b2b0d0faa..5b2dece969155 100644 --- a/roofit/histfactory/src/hist2workspace.cxx +++ b/roofit/histfactory/src/hist2workspace.cxx @@ -46,11 +46,16 @@ // help="disable the binned fit optimization used in HistFactory since ROOT 6.28", // action="store_true", // ) +// parser.add_argument( +// "-legacy_model_structure", +// help="build the channels from several RooFit objects per histogram, as before ROOT 6.44", +// action="store_true", +// ) // return parser // ``` constexpr static const char kCommandLineOptionsHelp[] = R"RAW( -usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] +usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] [-legacy_model_structure] hist2workspace is a utility to create RooFit/RooStats workspace from histograms @@ -59,6 +64,8 @@ hist2workspace is a utility to create RooFit/RooStats workspace from histograms -v switch HistFactory message stream to INFO level -vv switch HistFactory message stream to DEBUG level -disable_binned_fit_optimization disable the binned fit optimization used in HistFactory since ROOT 6.28 + -legacy_model_structure build the channels from several RooFit objects per histogram, as before + ROOT 6.44 )RAW"; @@ -102,6 +109,7 @@ namespace RooStats { * -v Switch HistFactory message stream to INFO level. * -vv Switch HistFactory message stream to DEBUG level. * -disable_binned_fit_optimization Disable the binned fit optimization used in HistFactory since ROOT 6.28. + * -legacy_model_structure Build the channels from several RooFit objects per histogram, as before ROOT 6.44. */ int main(int argc, char** argv) { @@ -145,6 +153,11 @@ int main(int argc, char** argv) { continue; } + if(input == "-legacy_model_structure") { + cfg.useHistFactoryPdf = false; + continue; + } + driverArg = argv[i]; } diff --git a/roofit/histfactory/test/CMakeLists.txt b/roofit/histfactory/test/CMakeLists.txt index 84e9553995069..ff541aeb57287 100644 --- a/roofit/histfactory/test/CMakeLists.txt +++ b/roofit/histfactory/test/CMakeLists.txt @@ -20,6 +20,11 @@ if(clad) target_compile_definitions(testHistFactory PUBLIC TEST_CODEGEN_AD) endif(clad) +ROOT_ADD_GTEST(testRooHistFactoryPdf testRooHistFactoryPdf.cxx LIBRARIES RooFitCore RooFit RooStats HistFactory) +if(clad) + target_compile_definitions(testRooHistFactoryPdf PUBLIC TEST_CODEGEN_AD) +endif(clad) + ROOT_ADD_GTEST(testParamHistFunc testParamHistFunc.cxx LIBRARIES RooFitCore HistFactory) ROOT_ADD_GTEST(testPiecewiseInterpolation testPiecewiseInterpolation.cxx LIBRARIES RooFitCore HistFactory) ROOT_ADD_GTEST(testHistFactoryPlotting testHistFactoryPlotting.cxx LIBRARIES RooFitCore HistFactory) diff --git a/roofit/histfactory/test/testHistFactory.cxx b/roofit/histfactory/test/testHistFactory.cxx index 9c8e2b804925e..0b1846f263be2 100644 --- a/roofit/histfactory/test/testHistFactory.cxx +++ b/roofit/histfactory/test/testHistFactory.cxx @@ -4,6 +4,8 @@ #include #include +#include +#include #ifdef HISTFACTORY_XML #include #endif @@ -363,7 +365,7 @@ TEST_P(HFFixture, ModelProperties) auto simPdf = dynamic_cast(ws->pdf("simPdf")); ASSERT_NE(simPdf, nullptr); - auto channelPdf = dynamic_cast(ws->pdf("channel1_model")); + auto channelPdf = dynamic_cast(ws->pdf("channel1_model")); ASSERT_NE(channelPdf, nullptr); auto obs = dynamic_cast(ws->var("obs_x_channel1")); @@ -397,36 +399,43 @@ TEST_P(HFFixture, ModelProperties) } } - // Check that sub models depend on their systematic uncertainties. - for (auto &subModelName : std::initializer_list{"signal_channel1_shapes", "background1_channel1_shapes", - "background2_channel1_shapes"}) { - auto subModel = ws->function(subModelName); - ASSERT_NE(subModel, nullptr) << "Unable to retrieve sub model with name " << subModelName; - if (subModelName.find("signal") != std::string::npos) { - EXPECT_FALSE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); - EXPECT_FALSE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); + // Check that the samples depend on their systematic uncertainties. + auto sample = [&](std::string const &name) -> RooHistFactorySample & { + auto *out = dynamic_cast(ws->function((name + "_channel1_shapes").c_str())); + if (!out) { + throw std::runtime_error("Unable to retrieve sample " + name); + } + EXPECT_EQ(out, channelPdf->sample(name)) << name; + return *out; + }; + EXPECT_EQ(channelPdf->numSamples(), 3u); + for (auto &sampleName : std::initializer_list{"signal", "background1", "background2"}) { + RooHistFactorySample &subModel = sample(sampleName); + if (sampleName == "signal") { + EXPECT_FALSE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); + EXPECT_FALSE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); } else { - EXPECT_TRUE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); - EXPECT_TRUE(subModel->dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); + EXPECT_TRUE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_0"))); + EXPECT_TRUE(subModel.dependsOn(*ws->var("gamma_stat_channel1_bin_1"))); } } - EXPECT_TRUE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("SigXsecOverSM"))); - EXPECT_FALSE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("SigXsecOverSM"))); - EXPECT_FALSE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("SigXsecOverSM"))); + EXPECT_TRUE(sample("signal").dependsOn(*ws->var("SigXsecOverSM"))); + EXPECT_FALSE(sample("background1").dependsOn(*ws->var("SigXsecOverSM"))); + EXPECT_FALSE(sample("background2").dependsOn(*ws->var("SigXsecOverSM"))); if (makeModelMode == MakeModelMode::OverallSyst) { - EXPECT_FALSE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst2"))); - EXPECT_FALSE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst3"))); - EXPECT_FALSE(ws->function("signal_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst4"))); + EXPECT_FALSE(sample("signal").dependsOn(*ws->var("alpha_syst2"))); + EXPECT_FALSE(sample("signal").dependsOn(*ws->var("alpha_syst3"))); + EXPECT_FALSE(sample("signal").dependsOn(*ws->var("alpha_syst4"))); - EXPECT_TRUE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst2"))); - EXPECT_TRUE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst3"))); - EXPECT_FALSE(ws->function("background1_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst4"))); + EXPECT_TRUE(sample("background1").dependsOn(*ws->var("alpha_syst2"))); + EXPECT_TRUE(sample("background1").dependsOn(*ws->var("alpha_syst3"))); + EXPECT_FALSE(sample("background1").dependsOn(*ws->var("alpha_syst4"))); - EXPECT_FALSE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst2"))); - EXPECT_TRUE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst3"))); - EXPECT_TRUE(ws->function("background2_channel1_scaleFactors")->dependsOn(*ws->var("alpha_syst4"))); + EXPECT_FALSE(sample("background2").dependsOn(*ws->var("alpha_syst2"))); + EXPECT_TRUE(sample("background2").dependsOn(*ws->var("alpha_syst3"))); + EXPECT_TRUE(sample("background2").dependsOn(*ws->var("alpha_syst4"))); } EXPECT_EQ(*mc->GetParametersOfInterest()->begin(), ws->var("SigXsecOverSM")); diff --git a/roofit/histfactory/test/testRooHistFactoryPdf.cxx b/roofit/histfactory/test/testRooHistFactoryPdf.cxx new file mode 100644 index 0000000000000..afecc7f2b0554 --- /dev/null +++ b/roofit/histfactory/test/testRooHistFactoryPdf.cxx @@ -0,0 +1,306 @@ +// Tests for RooHistFactoryPdf and RooHistFactorySample +// Authors: Jonas Rembser, CERN 2026 + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "../../roofitcore/test/gtest_wrapper.h" + +#include +#include + +using namespace RooStats::HistFactory; + +namespace { + +TH1 *makeHist(TRandom3 &rng, const char *name, int dim, double scale) +{ + double xbins[] = {0., 0.5, 1.7, 2.0}; + TH1 *h = dim == 1 ? static_cast(new TH1D(name, name, 3, xbins)) + : static_cast(new TH2D(name, name, 3, xbins, 2, 0, 1)); + h->SetDirectory(nullptr); + for (int i = 1; i <= h->GetNbinsX(); ++i) { + for (int j = 1; j <= h->GetNbinsY(); ++j) { + const int bin = h->GetBin(i, j); + const double v = scale * rng.Uniform(5, 50); + h->SetBinContent(bin, v); + h->SetBinError(bin, std::sqrt(v) * rng.Uniform(0.1, 0.5)); + } + } + return h; +} + +TH1 *varied(TRandom3 &rng, TH1 const *h, const char *name, double lo, double hi) +{ + auto *out = static_cast(h->Clone(name)); + out->SetDirectory(nullptr); + for (int b = 0; b < out->GetNcells(); ++b) { + out->SetBinContent(b, h->GetBinContent(b) * rng.Uniform(lo, hi)); + } + return out; +} + +/// A measurement with all types of modifiers, and with non-uniform bins. +std::unique_ptr makeMeasurement(int dim) +{ + TRandom3 rng{1337}; + auto meas = std::make_unique("meas", "meas"); + meas->SetPOI("mu"); + meas->SetLumi(1.0); + meas->SetLumiRelErr(0.05); + meas->AddLogNormSyst("lognorm", 0.1); + + Channel chan("ch"); + TH1 *data = makeHist(rng, "data", dim, 3.0); + for (int b = 0; b < data->GetNcells(); ++b) { + data->SetBinContent(b, std::round(data->GetBinContent(b))); + } + chan.SetData(data); + chan.SetStatErrorConfig(0.01, "Poisson"); + + Sample sig("signal"); + TH1 *hsig = makeHist(rng, "sig", dim, 1.0); + sig.SetHisto(hsig); + sig.AddNormFactor("mu", 1, 0, 5); + sig.AddOverallSys("syst1", 0.97, 1.03); + sig.AddOverallSys("lognorm", 0.9, 1.1); + for (const char *name : {"shape1", "shape2"}) { + HistoSys hs; + hs.SetName(name); + hs.SetHistoLow(varied(rng, hsig, (std::string{name} + "Lo").c_str(), 0.6, 1.0)); + hs.SetHistoHigh(varied(rng, hsig, (std::string{name} + "Hi").c_str(), 1.0, 1.4)); + sig.AddHistoSys(hs); + } + sig.ActivateStatError(); + chan.AddSample(sig); + + Sample bkg("background"); + TH1 *hbkg = makeHist(rng, "bkg", dim, 2.0); + bkg.SetHisto(hbkg); + bkg.AddOverallSys("syst2", 0.9, 1.2); + bkg.ActivateStatError(); + ShapeSys ss; + ss.SetName("bkgShape"); + TH1 *relErr = varied(rng, hbkg, "relErr", 0.0, 0.0); + for (int b = 0; b < relErr->GetNcells(); ++b) { + relErr->SetBinContent(b, rng.Uniform(0.02, 0.2)); + } + ss.SetErrorHist(relErr); + ss.SetConstraintType(Constraint::Poisson); + bkg.AddShapeSys(ss); + chan.AddSample(bkg); + + Sample free("free"); + free.SetHisto(makeHist(rng, "free", dim, 0.5)); + free.SetNormalizeByTheory(false); + free.AddShapeFactor("freeShape", 1.0, 0.0, 5.0); + chan.AddSample(free); + + meas->AddChannel(chan); + return meas; +} + +std::unique_ptr makeWorkspace(Measurement &meas, bool useHistFactoryPdf) +{ + HistoToWorkspaceFactoryFast::Configuration cfg; + cfg.useHistFactoryPdf = useHistFactoryPdf; + cfg.createPerRegionWorkspaces = false; + return std::unique_ptr{HistoToWorkspaceFactoryFast::MakeCombinedModel(meas, cfg)}; +} + +/// Change the parameters in both workspaces to the same values, close to the current ones. The constrained parameters +/// are changed by at most their uncertainty, so that the constraint terms don't underflow to zero. +void setParameters(TRandom3 &rng, RooWorkspace &ws1, RooWorkspace &ws2) +{ + for (RooAbsArg *arg : ws1.allVars()) { + auto *var1 = static_cast(arg); + auto *var2 = ws2.var(var1->GetName()); + if (var1->isConstant() || !var2) { + continue; + } + const double shift = var1->getError() > 0. ? var1->getError() : 0.3; + const double val = std::clamp(var1->getVal() + rng.Uniform(-shift, shift), var1->getMin(), var1->getMax()); + var1->setVal(val); + var2->setVal(val); + } +} + +} // namespace + +class RooHistFactoryPdfTest : public testing::TestWithParam {}; + +/// The new model structure gives the same values and likelihoods as the one before ROOT 6.44. +TEST_P(RooHistFactoryPdfTest, SameAsLegacyStructure) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + const int dim = GetParam(); + auto meas = makeMeasurement(dim); + auto wsOld = makeWorkspace(*meas, false); + auto wsNew = makeWorkspace(*meas, true); + + auto *pdfOld = wsOld->pdf("ch_model"); + auto *pdfNew = dynamic_cast(wsNew->pdf("ch_model")); + ASSERT_NE(pdfNew, nullptr); + EXPECT_EQ(pdfNew->numSamples(), 3u); + auto *mcOld = static_cast(wsOld->obj("ModelConfig")); + auto *mcNew = static_cast(wsNew->obj("ModelConfig")); + EXPECT_EQ(mcOld->GetNuisanceParameters()->size(), mcNew->GetNuisanceParameters()->size()); + EXPECT_EQ(mcOld->GetGlobalObservables()->size(), mcNew->GetGlobalObservables()->size()); + + RooArgSet obsOld; + RooArgSet obsNew; + pdfOld->getObservables(mcOld->GetObservables(), obsOld); + pdfNew->getObservables(mcNew->GetObservables(), obsNew); + + TRandom3 rng{42}; + for (int iPoint = 0; iPoint < 4; ++iPoint) { + if (iPoint > 0) { + setParameters(rng, *wsOld, *wsNew); + } + RooDataHist bins{"bins", "bins", obsOld}; + for (int i = 0; i < bins.numEntries(); ++i) { + obsOld.assign(*bins.get(i)); + obsNew.assign(*bins.get(i)); + // The same operations in the same order give the same values. + EXPECT_DOUBLE_EQ(pdfNew->getVal(), pdfOld->getVal()) << "bin " << i; + EXPECT_NEAR(pdfNew->getVal(obsNew), pdfOld->getVal(obsOld), 1e-12 * pdfOld->getVal(obsOld)) << "bin " << i; + } + EXPECT_NEAR(pdfNew->expectedEvents(&obsNew), pdfOld->expectedEvents(&obsOld), + 1e-12 * pdfOld->expectedEvents(&obsOld)); + + for (bool binned : {true, false}) { + pdfOld->setAttribute("BinnedLikelihood", binned); + pdfNew->setAttribute("BinnedLikelihood", binned); + using RooFit::GlobalObservables; + std::unique_ptr nllOld{ + mcOld->GetPdf()->createNLL(*wsOld->data("obsData"), GlobalObservables(*mcOld->GetGlobalObservables()))}; + std::unique_ptr nllNew{ + mcNew->GetPdf()->createNLL(*wsNew->data("obsData"), GlobalObservables(*mcNew->GetGlobalObservables()))}; + ASSERT_TRUE(std::isfinite(nllOld->getVal())) << "point " << iPoint << ", binned = " << binned; + EXPECT_NEAR(nllNew->getVal(), nllOld->getVal(), 1e-12 * std::abs(nllOld->getVal())) + << "point " << iPoint << ", binned = " << binned; + } + } +} + +#ifdef TEST_CODEGEN_AD +/// The likelihood with the code generation backend is the same as with the cpu backend. +TEST_P(RooHistFactoryPdfTest, Codegen) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto meas = makeMeasurement(GetParam()); + auto ws = makeWorkspace(*meas, true); + auto *mc = static_cast(ws->obj("ModelConfig")); + + TRandom3 rng{7}; + for (bool binned : {true, false}) { + ws->pdf("ch_model")->setAttribute("BinnedLikelihood", binned); + using RooFit::GlobalObservables; + std::unique_ptr nllCpu{mc->GetPdf()->createNLL( + *ws->data("obsData"), GlobalObservables(*mc->GetGlobalObservables()), RooFit::EvalBackend::Cpu())}; + std::unique_ptr nllCodegen{mc->GetPdf()->createNLL( + *ws->data("obsData"), GlobalObservables(*mc->GetGlobalObservables()), RooFit::EvalBackend::Codegen())}; + for (int iPoint = 0; iPoint < 3; ++iPoint) { + if (iPoint > 0) { + setParameters(rng, *ws, *ws); + } + ASSERT_TRUE(std::isfinite(nllCpu->getVal())) << "point " << iPoint << ", binned = " << binned; + EXPECT_NEAR(nllCodegen->getVal(), nllCpu->getVal(), 1e-10 * std::abs(nllCpu->getVal())) + << "point " << iPoint << ", binned = " << binned; + } + ws->loadSnapshot("NominalParamValues"); + } +} +#endif + +/// The samples and their data survive the writing to a file, and the data is shared between clones. +TEST_P(RooHistFactoryPdfTest, PersistenceAndClones) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto meas = makeMeasurement(GetParam()); + auto ws = makeWorkspace(*meas, true); + + const std::string fileName = "testRooHistFactoryPdf_" + std::to_string(GetParam()) + ".root"; + { + TFile file(fileName.c_str(), "RECREATE"); + file.WriteObject(ws.get(), "ws"); + } + std::unique_ptr file{TFile::Open(fileName.c_str())}; + std::unique_ptr wsRead{file->Get("ws")}; + ASSERT_NE(wsRead, nullptr); + + auto *pdf = dynamic_cast(ws->pdf("ch_model")); + auto *pdfRead = dynamic_cast(wsRead->pdf("ch_model")); + ASSERT_NE(pdfRead, nullptr); + ASSERT_EQ(pdfRead->numSamples(), pdf->numSamples()); + + TRandom3 rng{3}; + setParameters(rng, *ws, *wsRead); + for (std::size_t i = 0; i < pdf->numSamples(); ++i) { + RooHistFactorySample const &sample = pdf->sample(i); + RooHistFactorySample const &sampleRead = pdfRead->sample(i); + EXPECT_EQ(sampleRead.sampleName(), sample.sampleName()); + EXPECT_EQ(sampleRead.data().contents, sample.data().contents); + EXPECT_EQ(sampleRead.data().histoSysHi, sample.data().histoSysHi); + EXPECT_EQ(sampleRead.data().edges, sample.data().edges); + std::vector values(sample.numBins()); + std::vector valuesRead(sample.numBins()); + sample.binValues(values, false); + sampleRead.binValues(valuesRead, false); + EXPECT_EQ(valuesRead, values) << sample.sampleName(); + } + + // A clone shares the data, until its interpolation codes are changed. + auto const &signal = *pdf->sample("signal"); + std::unique_ptr clone{static_cast(signal.clone())}; + EXPECT_EQ(&clone->data(), &signal.data()); + clone->setHistoSysInterpCode(*ws->var("alpha_shape1"), 0); + EXPECT_NE(&clone->data(), &signal.data()); + EXPECT_EQ(clone->data().histoSysCodes, (std::vector{0, 4})); + EXPECT_EQ(signal.data().histoSysCodes, (std::vector{4, 4})); +} + +/// Integrals over ranges that don't align with the bins take the fraction of the bins in the range. +TEST(RooHistFactoryPdf, RangedIntegral) +{ + RooRealVar x{"x", "x", 0, 4}; + x.setBins(4); + RooRealVar mu{"mu", "mu", 2.0}; + RooHistFactorySample::Spec spec; + spec.sampleName = "sample"; + spec.contents = {1., 2., 3., 4.}; + spec.normFactors.push_back({"mu", &mu}); + RooHistFactorySample sample{"sample", "sample", RooArgList{x}, spec}; + RooHistFactoryPdf pdf{"pdf", "pdf", RooArgList{sample}}; + + x.setRange("r", 0.5, 2.25); + std::unique_ptr integral{sample.createIntegral(x, "r")}; + EXPECT_DOUBLE_EQ(integral->getVal(), mu.getVal() * (0.5 * 1. + 2. + 0.25 * 3.)); + std::unique_ptr full{sample.createIntegral(x)}; + EXPECT_DOUBLE_EQ(full->getVal(), mu.getVal() * 10.); + RooArgSet xSet{x}; + EXPECT_DOUBLE_EQ(pdf.expectedEvents(&xSet), mu.getVal() * 10.); + x.setVal(2.5); + RooArgSet normSet{x}; + EXPECT_DOUBLE_EQ(pdf.getVal(normSet), 3. / 10.); +} + +INSTANTIATE_TEST_SUITE_P(RooHistFactoryPdf, RooHistFactoryPdfTest, testing::Values(1, 2), + [](testing::TestParamInfo const ¶mInfo) { + return std::to_string(paramInfo.param) + "D"; + }); diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index aac0e3329f720..17c518ac014b6 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -96,6 +96,10 @@ class RooJSONFactoryWSTool { /// their data set of one entry in the combined data set, so that the RooSimultaneous is the sampling /// distribution of all measurements. This option and the form of the resulting workspace may still change. bool importConstraintsAsChannels = false; + /// Import each `histfactory_dist` as a RooHistFactoryPdf of RooHistFactorySample objects. If false, or if a + /// channel has custom modifiers that depend on the observables, the channel is imported as before ROOT 6.44, + /// with several RooFit objects for each histogram. + bool useHistFactoryPdf = true; }; static Config &config(); diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 5a7a711bb6e18..139c69b660f2c 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -393,13 +395,10 @@ void configureGammasWithConstraints(RooArgList const &gammas, std::vector const &parnames, std::vector const &vals, - HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, - double gammaMax, double minSigma, std::size_t nBins, std::string const &context, - bool configureGammas = true) +// The gamma parameters of a modifier that scales each bin, in the order of the bins (last axis fastest). +RooArgList createGammas(RooJSONFactoryWSTool &tool, std::string const &sysname, + std::vector const &parnames, RooArgSet const &observables, double gammaMax, + std::size_t nBins) { RooWorkspace &ws = *tool.workspace(); @@ -411,9 +410,28 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, gammas.add(existing ? *existing : getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); } + // Documents before HS3 0.3 didn't define the order of the parameters of multi-dimensional modifiers. + checkParamHistFuncOrderIsDefined(tool.hs3Version(), observables, "the modifier '" + sysname + "'"); + return gammas; +} + +void constrainGammas(RooJSONFactoryWSTool &tool, std::string const &sysname, RooArgList const &gammas, + std::vector const &vals, HS3::GammaConstraint const &constraint, RooArgSet &constraints, + double minSigma, std::string const &context, bool configureGammas); + +// Create the ParamHistFunc of the gamma parameters, together with their constraints, see constrainGammas(). +ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, std::string const &sysname, + std::vector const &parnames, std::vector const &vals, + HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, + double gammaMax, double minSigma, std::size_t nBins, std::string const &context, + bool configureGammas = true) +{ + RooWorkspace &ws = *tool.workspace(); + + RooArgList gammas = createGammas(tool, sysname, parnames, observables, gammaMax, nBins); + // The gammas and `vals` are given in the order of HS3 (last axis fastest), which is also the order in which they // are paired below. ParamHistFunc expects its parameters in TH1 order. - checkParamHistFuncOrderIsDefined(tool.hs3Version(), observables, "the modifier '" + sysname + "'"); RooArgList gammasInTH1Order = toParamHistFuncOrder(gammas, observables); // Shape modifiers of the same name in a channel share the function, so they need the same parameters. @@ -427,6 +445,17 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, } auto &phf = tool.wsEmplace(phfname, observables, gammasInTH1Order); + constrainGammas(tool, sysname, gammas, vals, constraint, constraints, minSigma, context, configureGammas); + + return phf; +} + +// Create the constraints of the gamma parameters. With a constraint type, RooFit creates them from the relative +// uncertainties `vals`, like HistFactory. +void constrainGammas(RooJSONFactoryWSTool &tool, std::string const &sysname, RooArgList const &gammas, + std::vector const &vals, HS3::GammaConstraint const &constraint, RooArgSet &constraints, + double minSigma, std::string const &context, bool configureGammas) +{ auto addGammaConstraints = [&](Constraint::Type type) { for (auto const &term : createGammaConstraints(gammas, vals, minSigma, type).constraints) { constraints.add(tool.wsImport(*term)); @@ -451,7 +480,7 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, } if (vals.empty()) { - return phf; + return; } std::visit(HS3::overloaded{ @@ -478,8 +507,6 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, }, }, constraint); - - return phf; } // The relative uncertainty that the constraint distribution of a gamma encodes, see gammaRelUncertainty(). Constraints @@ -659,6 +686,134 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample) } } +// Collects the bin contents and modifiers of a sample for a RooHistFactorySample, creating the parameters and their +// constraints like buildSample(). +RooHistFactorySample::Spec buildSampleSpec(ChannelContext const &ctx, HS3::Sample const &sample, + RooArgList const &statGammas, std::map &shapeGammas) +{ + using BinFactorType = RooHistFactorySample::BinFactorType; + + RooJSONFactoryWSTool &tool = ctx.tool; + std::string const &sampleName = sample.name; + + RooHistFactorySample::Spec spec; + spec.sampleName = sampleName; + spec.contents = sample.contents; + if (sample.errors) { + spec.errors = *sample.errors; + } + + // The validation made sure that the channel has a "default_interpolation" if a modifier doesn't give its own. + auto interpolationCode = [&](std::optional const &interpolation, + InterpolationClass interpolationClass, const char *type, std::string const &name) { + return codeFromInterpolation(interpolation ? *interpolation : ctx.dist.defaultInterpolation.value(), + interpolationClass, + std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + + sampleName + "' of channel '" + ctx.channelName + "'"); + }; + + // Like in buildSample(), the parameters of the staterror come first. + for (HS3::Modifier const &modifier : sample.modifiers) { + if (auto const *statError = std::get_if(&modifier)) { + spec.binFactors.push_back({statError->name, BinFactorType::StatError, statGammas}); + break; + } + } + + for (HS3::Modifier const &modifier : sample.modifiers) { + std::visit( + HS3::overloaded{ + [&](HS3::StatError const &) { + // see above + }, + [&](HS3::NormFactor const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 1., -3, 5, mod.name); + // Like in buildSample(), an error that is already set is kept. + if (auto *var = dynamic_cast(¶m); var && !var->hasError()) { + var->setError(0.0); + } + spec.normFactors.push_back({mod.name, ¶m}); + // for norm factors, constraints are optional + addConstraint(tool, mod.constraint, param, 1.0, "normfactor", mod.name, sampleName, ctx.constraints); + }, + [&](HS3::NormSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + const int interp = + interpolationCode(mod.interpolation, InterpolationClass::Flexible, "normsys", mod.name); + double low = mod.lo; + double high = mod.hi; + // The same cutoff of variations below zero as in buildSample(). + if (interp == 4 && low <= 0) + low = std::numeric_limits::epsilon(); + if (interp == 4 && high <= 0) + high = std::numeric_limits::epsilon(); + spec.normSys.push_back({mod.name, ¶m, low, high, interp}); + addSysConstraint(tool, mod.constraint, param, "normsys", mod.name, sampleName, ctx.constraints); + }, + [&](HS3::HistoSys const &mod) { + RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); + spec.histoSys.push_back( + {mod.name, ¶m, mod.lo.contents, mod.hi.contents, + interpolationCode(mod.interpolation, InterpolationClass::Piecewise, "histosys", mod.name)}); + addSysConstraint(tool, mod.constraint, param, "histosys", mod.name, sampleName, ctx.constraints); + }, + [&](HS3::ShapeSys const &mod) { + const bool constrained = std::visit( + HS3::overloaded{[](HS3::Unconstrained) { return false; }, [](HS3::ConstantGammas) { return false; }, + [&](HS3::ConstraintType) { return !mod.vals.empty(); }, + [](HS3::GammaConstraintPdfs const &constraints) { return anyNamed(constraints); }}, + mod.constraint); + RooArgList gammas = createGammas(tool, mod.name, mod.parameters, ctx.observables, + defaultShapeSysGammaMax, sample.contents.size()); + // Shape modifiers of the same name in a channel share their parameters. + auto found = shapeGammas.find(mod.name); + if (found == shapeGammas.end()) { + shapeGammas.emplace(mod.name, gammas); + } else { + bool same = found->second.size() == gammas.size(); + for (std::size_t i = 0; same && i < gammas.size(); ++i) { + same = &found->second[i] == &gammas[i]; + } + if (!same) { + RooJSONFactoryWSTool::error("the modifier '" + mod.name + "' has other parameters than another " + + "modifier of this name in the channel"); + } + } + constrainGammas(tool, mod.name, gammas, mod.vals, mod.constraint, ctx.constraints, minShapeUncertainty, + "the shapesys modifier '" + mod.name + "' of sample '" + sampleName + "'", + /*configureGammas=*/true); + spec.binFactors.push_back( + {mod.name, constrained ? BinFactorType::ShapeSys : BinFactorType::ShapeFactor, gammas}); + }, + [&](HS3::CustomModifier const &mod) { + // The function can come later in the document. + RooAbsReal *obj = tool.request(mod.name, ctx.channelName); + spec.normFactors.push_back({mod.name, obj}); + }, + }, + modifier); + } + + return spec; +} + +// RooHistFactorySample doesn't support custom modifiers that depend on the observables. Channels with such modifiers +// are imported with the structure before ROOT 6.44. +bool hasObservableDependentModifiers(RooJSONFactoryWSTool &tool, HS3::HistFactoryDist const &dist, + RooArgSet const &observables, std::string const &channelName) +{ + for (auto const &sample : dist.samples) { + for (auto const &modifier : sample.modifiers) { + if (auto const *custom = std::get_if(&modifier)) { + if (tool.request(custom->name, channelName)->dependsOn(observables)) { + return true; + } + } + } + } + return false; +} + // The constraint of the gamma parameters of the staterror modifiers. Without one, the gamma parameters are free. HS3::GammaConstraint statErrorConstraint(HS3::HistFactoryDist const &dist) { @@ -1359,13 +1514,177 @@ bool isValidChannel(ExportedChannel const &channel, std::string const &pdfname) return !HS3::hasErrors(diagnostics); } +// Reads a RooHistFactoryPdf, which has the modifiers of its samples explicitly. +ExportedChannel readHistFactoryPdf(RooJSONFactoryWSTool *tool, const std::string &pdfname, RooHistFactoryPdf const &pdf, + std::vector const &channelConstraints) +{ + using BinFactorType = RooHistFactorySample::BinFactorType; + + ExportedChannel channel; + RooWorkspace *ws = tool->workspace(); + + channel.name = pdfname; + erasePrefix(channel.name, "model_"); + eraseSuffix(channel.name, "_model"); + channel.observables = std::make_unique(pdf.observables()); + + // Like in readChannel(), the modifiers name only the constraint terms of the channel. + auto constraintTerm = [&](RooAbsPdf const *pdf) -> std::optional { + if (std::find(channelConstraints.begin(), channelConstraints.end(), pdf) == channelConstraints.end()) { + return std::nullopt; + } + channel.constraints.emplace(pdf->GetName(), pdf); + return HS3::DistributionRef{pdf->GetName()}; + }; + auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { + auto ref = constraintTerm(pdf); + return ref ? HS3::Constraint{HS3::ConstraintPdf{*ref}} : HS3::Unconstrained{}; + }; + // The constraints of the gammas, in the order of the bins. If none is named, the gammas are free. + auto gammaConstraints = [&](std::vector const &gammas, bool alsoByName) -> HS3::GammaConstraint { + HS3::GammaConstraintPdfs pdfs; + for (RooAbsReal *g : gammas) { + RooAbsPdf *constraintPdf = findConstraint(g, channelConstraints); + if (!constraintPdf && alsoByName) { + constraintPdf = ws->pdf(constraintName(g->GetName())); + } + pdfs.pdfs.push_back(constraintTerm(constraintPdf)); + } + if (anyNamed(pdfs)) { + return pdfs; + } + return HS3::Unconstrained{}; + }; + + for (std::size_t iSample = 0; iSample < pdf.numSamples(); ++iSample) { + RooHistFactorySample const &hfSample = pdf.sample(iSample); + RooHistFactorySample::Data const &data = hfSample.data(); + auto param = [&](int index) { return &static_cast(hfSample.parameters()[index]); }; + const std::size_t nBins = data.numBins(); + + HS3::Sample sample; + sample.name = data.sampleName; + sample.contents = data.contents; + if (!data.errors.empty()) { + sample.errors = data.errors; + } + + // The modifiers by type, which are written in this order, like for the models before ROOT 6.44. + std::vector normFactors; + std::vector normSys; + std::vector histoSys; + std::vector shapeSys; + std::vector custom; + std::optional statError; + + for (std::size_t i = 0; i < data.normFactorParams.size(); ++i) { + RooAbsReal *par = param(data.normFactorParams[i]); + if (auto constVar = dynamic_cast(par)) { + if (constVar->getVal() != 1.) { + normFactors.push_back({constVar->GetName(), {constVar->GetName()}, HS3::Unconstrained{}}); + } + } else if (dynamic_cast(par)) { + normFactors.push_back( + {par->GetName(), {par->GetName()}, constraint(findConstraint(par, channelConstraints))}); + } else { + custom.push_back({par->GetName()}); + channel.customModifiers.push_back(par); + } + } + + for (std::size_t i = 0; i < data.normSysParams.size(); ++i) { + RooAbsReal *par = param(data.normSysParams[i]); + RooAbsPdf const *constraintPdf = findConstraint(par, channelConstraints); + checkModifierParameter(*par, constraintPdf); + const std::string context = "normsys modifier '" + data.normSysNames[i] + "' in sample '" + sample.name + + "' of channel '" + channel.name + "'"; + normSys.push_back({data.normSysNames[i], + {par->GetName()}, + data.normSysHi[i], + data.normSysLo[i], + constraint(constraintPdf), + interpolationFromCode(data.normSysCodes[i], InterpolationClass::Flexible, context)}); + } + + for (std::size_t i = 0; i < data.histoSysParams.size(); ++i) { + RooAbsReal *par = param(data.histoSysParams[i]); + RooAbsPdf const *constraintPdf = findConstraint(par, channelConstraints); + checkModifierParameter(*par, constraintPdf); + const std::string context = "histosys modifier '" + data.histoSysNames[i] + "' in sample '" + sample.name + + "' of channel '" + channel.name + "'"; + auto variation = [&](std::vector const &all) { + return HS3::HistoSysVariation{{all.begin() + i * nBins, all.begin() + (i + 1) * nBins}, std::nullopt}; + }; + histoSys.push_back({data.histoSysNames[i], + {par->GetName()}, + variation(data.histoSysHi), + variation(data.histoSysLo), + constraint(constraintPdf), + interpolationFromCode(data.histoSysCodes[i], InterpolationClass::Piecewise, context)}); + } + + for (std::size_t i = 0; i < data.binFactorNames.size(); ++i) { + std::vector gammas; + for (std::size_t iBin = 0; iBin < nBins; ++iBin) { + gammas.push_back(param(data.binFactorParams[i * nBins + iBin])); + } + if (static_cast(data.binFactorTypes[i]) == BinFactorType::StatError) { + statError.emplace(); + statError->name = data.binFactorNames[i]; + for (std::size_t iBin = 0; iBin < nBins; ++iBin) { + RooAbsReal *g = gammas[iBin]; + statError->parameters.push_back({g->GetName()}); + // To reconstruct the statistical uncertainties if the sample doesn't have them. + channel.totYield[iBin] += sample.contents[iBin]; + channel.totYield2[iBin] += sample.contents[iBin] * sample.contents[iBin]; + if (RooAbsPdf *constraintPdf = findConstraint(g, channelConstraints)) { + channel.relErrors[iBin] = constraintRelError(*constraintPdf, *g); + } + } + statError->constraint = gammaConstraints(gammas, false); + } else { + HS3::ShapeSys &sys = shapeSys.emplace_back(); + sys.name = data.binFactorNames[i]; + for (RooAbsReal *g : gammas) { + sys.parameters.push_back({g->GetName()}); + } + // Both are written as "shapefactor": the constrained parameters name their constraint distribution in + // "constraints", the others are free. + sys.constraint = gammaConstraints(gammas, true); + } + } + + sortByName(normFactors); + sortByName(normSys); + sortByName(histoSys); + sortByName(shapeSys); + auto append = [&](auto &modifiers) { + sample.modifiers.insert(sample.modifiers.end(), std::make_move_iterator(modifiers.begin()), + std::make_move_iterator(modifiers.end())); + }; + append(normFactors); + append(normSys); + append(histoSys); + append(shapeSys); + append(custom); + if (statError) { + sample.modifiers.push_back(std::move(*statError)); + } + + channel.dist.samples.push_back(std::move(sample)); + } + + sortByName(channel.dist.samples); + return channel; +} + // The statistical uncertainties of the samples are reconstructed from the constraints of the gamma parameters, with // the (arbitrary) convention that all samples have the same relative uncertainty. void configureStatError(ExportedChannel &channel) { for (HS3::Sample &sample : channel.dist.samples) { - if (std::none_of(sample.modifiers.begin(), sample.modifiers.end(), - [](auto const &m) { return std::holds_alternative(m); })) { + if (sample.errors || std::none_of(sample.modifiers.begin(), sample.modifiers.end(), + [](auto const &m) { return std::holds_alternative(m); })) { continue; } std::vector errors(sample.contents.size()); @@ -1522,20 +1841,24 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname return false; } - for (RooAbsArg *sample : sumpdf->funcList()) { - if (!dynamic_cast(sample) && !dynamic_cast(sample)) { - return false; + ExportedChannel channel; + if (auto *hfPdf = dynamic_cast(sumpdf)) { + channel = readHistFactoryPdf(tool, pdfname, *hfPdf, constraints); + } else { + for (RooAbsArg *sample : sumpdf->funcList()) { + if (!dynamic_cast(sample) && !dynamic_cast(sample)) { + return false; + } } - } - - ExportedChannel channel = readChannel(tool, pdfname, sumpdf, constraints); - // Writing a formula as several modifiers must not be what makes the channel invalid, e.g. because a parameter that - // occurs twice in the formula becomes two normfactors of the same name. If it is, the formulas are written as - // custom modifiers instead. - if (channel.decomposedFormulas && !isValidChannel(channel, pdfname)) { - ExportedChannel plain = readChannel(tool, pdfname, sumpdf, constraints, /*decomposeFormulas=*/false); - if (isValidChannel(plain, pdfname)) { - channel = std::move(plain); + channel = readChannel(tool, pdfname, sumpdf, constraints); + // Writing a formula as several modifiers must not be what makes the channel invalid, e.g. because a parameter + // that occurs twice in the formula becomes two normfactors of the same name. If it is, the formulas are written + // as custom modifiers instead. + if (channel.decomposedFormulas && !isValidChannel(channel, pdfname)) { + ExportedChannel plain = readChannel(tool, pdfname, sumpdf, constraints, /*decomposeFormulas=*/false); + if (isValidChannel(plain, pdfname)) { + channel = std::move(plain); + } } } HS3::HistFactoryDist &dist = channel.dist; @@ -1624,6 +1947,7 @@ const bool registered = []() { RooFit::JSONIO::registerImporter("histfactory_dist", true); RooFit::JSONIO::registerExporter(RooProdPdf::Class(), true); RooFit::JSONIO::registerExporter(RooRealSumPdf::Class(), true); + RooFit::JSONIO::registerExporter(RooHistFactoryPdf::Class(), true); return true; }(); @@ -1633,6 +1957,12 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c { RooArgSet observables = createObservables(dist.axes); + std::string channelName = name; + erasePrefix(channelName, "model_"); + + const bool useHistFactoryPdf = RooJSONFactoryWSTool::config().useHistFactoryPdf && + !hasObservableDependentModifiers(tool, dist, observables, channelName); + // The yields and their variances by bin, summed over the samples with a staterror modifier. std::vector sumW; std::vector sumW2; @@ -1659,10 +1989,8 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c } } - std::string channelName = name; - erasePrefix(channelName, "model_"); - RooAbsArg *mcStatObject = nullptr; + RooArgList statGammas; RooArgSet constraints; if (!sumW.empty()) { // The relative uncertainties. The gammas of bins without a positive yield are set constant later. @@ -1675,19 +2003,39 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c // range, the constness and the step width of the gammas come from the document, and nothing is derived from // the errors of the samples. Older documents relied on RooFit configuring the gammas from them. const bool configureGammas = tool.hs3Version() < HS3::Version{0, 3}; - mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, - statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, - dist.statError.relThreshold, errs.size(), - "the staterror modifier of channel '" + name + "'", configureGammas); + const std::string context = "the staterror modifier of channel '" + name + "'"; + if (useHistFactoryPdf) { + statGammas.add(createGammas(tool, "stat_" + channelName, gammaParnames, observables, defaultStatErrorGammaMax, + errs.size())); + constrainGammas(tool, "stat_" + channelName, statGammas, errs, statErrorConstraint(dist), constraints, + dist.statError.relThreshold, context, configureGammas); + } else { + mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, + statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, + dist.statError.relThreshold, errs.size(), context, configureGammas); + } } ChannelContext ctx{tool, dist, observables, name, channelName, mcStatObject, constraints}; RooArgList funcs; RooArgList coefs; + std::map shapeGammas; for (HS3::Sample const &sample : dist.samples) { + const std::string shapesName = name + "_" + sample.name + "_shapes"; + if (useHistFactoryPdf) { + if (tool.workspace()->arg(shapesName.c_str())) { + RooJSONFactoryWSTool::error("the object '" + shapesName + "' for sample '" + sample.name + + "' of channel '" + channelName + "' has the name of an existing object"); + } + RooArgList obsList{observables}; + funcs.add(tool.wsEmplace(shapesName, obsList, + buildSampleSpec(ctx, sample, statGammas, shapeGammas))); + funcs[funcs.size() - 1].SetTitle(sample.name.c_str()); + continue; + } buildSample(ctx, sample); - funcs.add(*tool.request(name + "_" + sample.name + "_shapes", name)); + funcs.add(*tool.request(shapesName, name)); coefs.add(*tool.request(name + "_" + sample.name + "_scaleFactors", name)); } @@ -1710,7 +2058,9 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c } // With constraints, the channel is the product of the sum of the samples and the constraints. - auto &sum = tool.wsEmplace(constraints.empty() ? name : channelName + "_model", funcs, coefs, true); + const std::string sumName = constraints.empty() ? name : channelName + "_model"; + auto &sum = useHistFactoryPdf ? tool.wsEmplace(sumName, funcs) + : tool.wsEmplace(sumName, funcs, coefs, true); sum.SetTitle(name.c_str()); // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation diff --git a/roofit/hs3/test/HS3TestUtils.h b/roofit/hs3/test/HS3TestUtils.h index cfa9b2d35e35d..6f46905b7a4ed 100644 --- a/roofit/hs3/test/HS3TestUtils.h +++ b/roofit/hs3/test/HS3TestUtils.h @@ -121,6 +121,10 @@ inline ConfigFlag importConstraintsAsChannels(bool value) { return {&RooJSONFactoryWSTool::Config::importConstraintsAsChannels, value}; } +inline ConfigFlag useHistFactoryPdf(bool value) +{ + return {&RooJSONFactoryWSTool::Config::useHistFactoryPdf, value}; +} /// Sets flags of RooJSONFactoryWSTool::config() for its lifetime. class ScopedConfig { diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 38e180e352700..8205eeae5a67d 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -47,6 +47,8 @@ #include #include #include +#include +#include #include #include #include @@ -91,6 +93,53 @@ std::string jsonString(RooFit::Detail::JSONTree &tree) return stream.str(); } +// The sample "sig" of the channel of HistFactoryDoc. +RooHistFactorySample *sigSample(RooWorkspace &ws) +{ + return object(ws, "model_channel0_sig_shapes"); +} + +// The interpolation codes of the NormSys or HistoSys modifiers of a sample, by parameter name. +std::map interpolationCodes(RooHistFactorySample const &sample, bool normSys) +{ + auto const &data = sample.data(); + auto const ¶ms = normSys ? data.normSysParams : data.histoSysParams; + auto const &codes = normSys ? data.normSysCodes : data.histoSysCodes; + std::map out; + for (std::size_t i = 0; i < params.size(); ++i) { + out[sample.parameters()[params[i]].GetName()] = codes[i]; + } + return out; +} + +// The type of the modifier of a sample that scales each bin with its own parameter, by name. +std::optional +binFactorType(RooHistFactorySample const &sample, std::string const &name) +{ + auto const &data = sample.data(); + for (std::size_t i = 0; i < data.binFactorNames.size(); ++i) { + if (data.binFactorNames[i] == name) { + return static_cast(data.binFactorTypes[i]); + } + } + return std::nullopt; +} + +// The parameters of the modifier of a sample that scales each bin with its own parameter, by name. +RooArgList binFactorParams(RooHistFactorySample const &sample, std::string const &name) +{ + auto const &data = sample.data(); + RooArgList params; + for (std::size_t i = 0; i < data.binFactorNames.size(); ++i) { + if (data.binFactorNames[i] == name) { + for (std::size_t iBin = 0; iBin < data.numBins(); ++iBin) { + params.add(sample.parameters()[data.binFactorParams[i * data.numBins() + iBin]]); + } + } + } + return params; +} + // Builds a single-bin-observable RooDataHist filled with the given bin contents. Returns an owning pointer so that it // can be handed to the RooHistFunc constructor that takes ownership (the const-reference overload keeps only an unowned // pointer, which would dangle for a temporary). @@ -1551,13 +1600,10 @@ TEST(RooFitHS3, HistFactoryInterpolationCodeMapping) EXPECT_FALSE(exported.has("distributions/model_channel0/samples/sig/modifiers/modifier/interpolation")); Imported roundTripped{exported}; - if (isNormSys) { - auto *interpolation = object(roundTripped, "sig_channel0_epsilon"); - EXPECT_EQ(interpolation->interpolationCodes(), std::vector{testCase.canonicalCode}); - } else { - auto *interpolation = object(roundTripped, "histoSys_model_channel0_sig"); - EXPECT_EQ(interpolation->interpolationCodes(), std::vector{testCase.canonicalCode}); - } + // The codes of NormSys are the ones of FlexibleInterpVar, and the codes of HistoSys the ones of + // PiecewiseInterpolation. + const std::map codes = interpolationCodes(*sigSample(roundTripped), isNormSys); + EXPECT_EQ(codes, (std::map{{"alpha_modifier", testCase.canonicalCode}})); } } @@ -1602,18 +1648,8 @@ TEST(RooFitHS3, HistFactoryInterpolationDefaultsAndOverrides) {"name": "shape_quadratic", "interpolation": {"type": "add", "in": "poly2", "out": "poly1"}}]}]})")); Imported roundTripped{exported}; - auto *shapeInterpolation = object(roundTripped, "histoSys_model_channel0_sig"); - ASSERT_EQ(shapeInterpolation->paramList().size(), 2u); - for (std::size_t i = 0; i < shapeInterpolation->paramList().size(); ++i) { - const std::string parameter = shapeInterpolation->paramList().at(i)->GetName(); - if (parameter == "alpha_shape_default") { - EXPECT_EQ(shapeInterpolation->interpolationCodes()[i], 4); - } else if (parameter == "alpha_shape_quadratic") { - EXPECT_EQ(shapeInterpolation->interpolationCodes()[i], 2); - } else { - FAIL() << "Unexpected histosys parameter " << parameter; - } - } + EXPECT_EQ(interpolationCodes(*sigSample(roundTripped), false), + (std::map{{"alpha_shape_default", 4}, {"alpha_shape_quadratic", 2}})); const std::string implicitModifiers = R"( {"name":"norm","parameter":"alpha_norm","type":"normsys","data":{"lo":0.8,"hi":1.2}}, @@ -1669,8 +1705,7 @@ std::vector structuredInterpolationYields(RooWorkspace &ws) ws.var("alpha_shape")->setVal(theta); for (int bin = 0; bin < 2; ++bin) { ws.var("x")->setBin(bin); - yields.push_back(ws.function("model_channel0_sig_shapes")->getVal(observables) * - ws.function("model_channel0_sig_scaleFactors")->getVal()); + yields.push_back(ws.function("model_channel0_sig_shapes")->getVal(observables)); } } return yields; @@ -1690,12 +1725,8 @@ INSTANTIATE_TEST_SUITE_P(HistFactoryStructuredInterpolation, RoundTrip, TEST(RooFitHS3, HistFactoryStructuredInterpolationRoundTrip) { Imported ws{structuredInterpolationJSON()}; - auto *norm = dynamic_cast(ws.function("sig_channel0_epsilon")); - auto *shape = dynamic_cast(ws.function("histoSys_model_channel0_sig")); - ASSERT_NE(norm, nullptr); - ASSERT_NE(shape, nullptr); - EXPECT_EQ(norm->interpolationCodes()[0], 4); - EXPECT_EQ(shape->interpolationCodes()[0], 2); + EXPECT_EQ(interpolationCodes(*sigSample(ws), true).at("alpha_norm"), 4); + EXPECT_EQ(interpolationCodes(*sigSample(ws), false).at("alpha_shape"), 2); } TEST(RooFitHS3, HistFactoryStructuredInterpolationValidation) @@ -1759,8 +1790,12 @@ TEST(RooFitHS3, HistFactoryZeroYieldBin) } } +// The RooFit objects of the HistFactory models before ROOT 6.44 don't store the names of the modifiers. The +// exporter names them after their parameters, and combines the modifiers that get the same name. TEST(RooFitHS3, HistFactoryDuplicateModifiersAreCombined) { + ScopedConfig config{{useHistFactoryPdf(false)}}; + // The two "poly" modifiers deliberately use implicit code 4 and explicit // code 5, which are semantic aliases in FlexibleInterpVar. Imported ws1{HistFactoryDoc{"0.1.90", R"( @@ -1849,9 +1884,55 @@ TEST(RooFitHS3, HistFactoryDuplicateModifiersAreCombined) EXPECT_TRUE(sentinelErrors.empty()) << sentinelErrors; } -// Duplicate modifiers that cannot be combined into one are rejected on export. +// A RooHistFactoryPdf keeps the names of the modifiers. Modifiers with the same name and type are combined in the +// export, and the others are kept as they are, even if they have the same parameter. +TEST(RooFitHS3, HistFactoryDuplicateModifiersRoundTrip) +{ + Imported ws1{HistFactoryDoc{"0.1.90", R"( + {"name": "norm", "parameter": "alpha_norm", "type": "normsys", "interpolation": 1, "data": {"lo": 0.8, "hi": 1.2}}, + {"name": "norm", "parameter": "alpha_norm", "type": "normsys", "interpolation": 1, "data": {"lo": 0.9, "hi": 1.1}}, + {"name": "shape_input_1", "parameter": "alpha_shape", "type": "histosys", + "data": {"lo": {"contents": [8.0, 18.0]}, "hi": {"contents": [12.0, 22.0]}}}, + {"name": "shape_input_2", "parameter": "alpha_shape", "type": "histosys", + "data": {"lo": {"contents": [9.0, 16.0]}, "hi": {"contents": [11.0, 24.0]}}})"}}; + + Doc exported{ws1}; + EXPECT_THAT(exported.warnings(), HasSubstr("combined 2 duplicate modifiers named 'norm' of type 'normsys'")); + EXPECT_THAT(exported.warnings(), Not(HasSubstr("of type 'histosys'"))); + EXPECT_EQ(exported.names("distributions/model_channel0/samples/sig/modifiers"), + (std::vector{"norm", "shape_input_1", "shape_input_2"})); + + Imported ws2{exported}; + RooHistFactorySample *sample = sigSample(ws2); + ASSERT_EQ(sample->data().normSysNames.size(), 1u); + EXPECT_DOUBLE_EQ(sample->data().normSysLo[0], 0.8 * 0.9); + EXPECT_DOUBLE_EQ(sample->data().normSysHi[0], 1.2 * 1.1); + EXPECT_EQ(sample->data().histoSysNames, (std::vector{"shape_input_1", "shape_input_2"})); + + for (double norm : {-0.6, 0.8, 1.3}) { + for (double shape : {-0.4, 0.35, 1.2}) { + for (int bin = 0; bin < 2; ++bin) { + auto prediction = [&](RooWorkspace &ws) { + ws.var("x")->setBin(bin); + ws.var("alpha_norm")->setVal(norm); + ws.var("alpha_shape")->setVal(shape); + return ws.function("model_channel0_sig_shapes")->getVal(); + }; + const double before = prediction(ws1); + EXPECT_NEAR(prediction(ws2), before, std::abs(before) * 1e-13) + << "bin=" << bin << ", norm=" << norm << ", shape=" << shape; + } + } + } +} + +// Duplicate modifiers that cannot be combined into one are rejected on export. The duplicates come from naming the +// modifiers after their parameters, which the exporter does for the RooFit objects of the HistFactory models before +// ROOT 6.44. A RooHistFactoryPdf keeps the names of the modifiers. TEST(RooFitHS3, HistFactoryIncompatibleDuplicateModifiers) { + ScopedConfig config{{useHistFactoryPdf(false)}}; + // Two normsys modifiers, "input_1" and "input_2", with the given further keys. auto normsys = [](std::string const &keys1, std::string const &keys2) { return R"({"name": "input_1", "type": "normsys", "data": {"lo": 0.8, "hi": 1.2}, )" + keys1 + R"(}, @@ -1929,12 +2010,13 @@ TEST(RooFitHS3, HistFactoryHistoSysOfTheSameNameInOlderDocuments) "there is another histosys modifier named 'shape' in the sample"); Imported ws{HistFactoryDoc{"0.2", modifiers}}; - auto *interp = object(ws, "histoSys_model_channel0_sig"); - ASSERT_EQ(interp->paramList().size(), 2u); - EXPECT_EQ(interp->paramList().at(0), interp->paramList().at(1)); - auto high = [&](int i, int bin) { - return static_cast(interp->highList().at(i))->dataHist().weight(bin); - }; + RooHistFactorySample *sample = sigSample(ws); + RooHistFactorySample::Data const &data = sample->data(); + EXPECT_EQ(data.histoSysNames, (std::vector{"shape", "shape"})); + ASSERT_EQ(data.histoSysParams.size(), 2u); + EXPECT_EQ(data.histoSysParams[0], data.histoSysParams[1]); + // The variations are stored one after the other. + auto high = [&](int i, int bin) { return data.histoSysHi[i * data.numBins() + bin]; }; EXPECT_DOUBLE_EQ(high(0, 1), 22.0); EXPECT_DOUBLE_EQ(high(1, 1), 24.0); @@ -1942,15 +2024,14 @@ TEST(RooFitHS3, HistFactoryHistoSysOfTheSameNameInOlderDocuments) Doc exported{ws}; EXPECT_EQ(exported.names("distributions/model_channel0/samples/sig/modifiers"), std::vector{"shape"}); Imported ws2{exported}; - auto *interp2 = object(ws2, "histoSys_model_channel0_sig"); - ASSERT_EQ(interp2->paramList().size(), 1u); - // Both functions give the same for a shifted parameter. (expectRoundTrip() cannot make this comparison: the - // variations of the first modifier and of the combined one have the same names.) + RooHistFactorySample *sample2 = sigSample(ws2); + EXPECT_EQ(sample2->data().histoSysNames, std::vector{"shape"}); + // Both samples give the same for a shifted parameter. ws.var("alpha_shape")->setVal(0.7); ws2.var("alpha_shape")->setVal(0.7); ws.var("x")->setBin(1); ws2.var("x")->setBin(1); - EXPECT_NEAR(interp->getVal(), interp2->getVal(), 1e-9 * interp->getVal()); + EXPECT_NEAR(sample->getVal(), sample2->getVal(), 1e-9 * sample->getVal()); } TEST(RooFitHS3, HistFactoryConstraintKeyMigration) @@ -2569,7 +2650,8 @@ TEST(RooFitHS3, HistFactoryStatErrorConstraint) // The constraints are exported as one array distribution that the staterror modifier references by name. Doc doc{ws}; EXPECT_FALSE(doc.has("distributions/model_channel0/staterror")); - const std::string modifier = "distributions/model_channel0/samples/sig/modifiers/staterror"; + // The name of the modifier in the document is kept. + const std::string modifier = "distributions/model_channel0/samples/sig/modifiers/mcstat"; EXPECT_FALSE(doc.has(modifier + "/constraint")); const std::vector constraints = doc.strings(modifier + "/constraints"); ASSERT_EQ(constraints.size(), 2u); @@ -2607,8 +2689,7 @@ TEST(RooFitHS3, HistFactoryMissingInterpolation) Imported ws{HistFactoryDoc{"0.2", modifiers}}; EXPECT_EQ(countOccurrences(ws.warnings, "implicit default of older RooFit versions"), 1u) << ws.warnings; - auto *norm = object(ws, "sig_channel0_epsilon"); - EXPECT_EQ(norm->interpolationCodes()[0], 4); + EXPECT_EQ(interpolationCodes(*sigSample(ws), true).at("alpha_norm"), 4); // RooFit always writes a "default_interpolation" if a modifier relies on it. Doc exported{ws}; @@ -3257,16 +3338,17 @@ TEST(RooFitHS3, HistFactoryArrayDistExport) // Opt-out: one constraint distribution per parameter, no array_dist at all. const std::string modifiers = "distributions/model_c1/samples/sig/modifiers/"; + // The staterror modifier, which has no name in the document, keeps the name that the import gave it. + const std::string staterror = modifiers + "c1_sig_syst_2"; Doc plain{ws, {exportArrayDist(false)}}; EXPECT_THAT(plain.json(), Not(HasSubstr("array_dist"))); EXPECT_TRUE(matches(plain, modifiers + "a", R"({"constraints": ["alpha_a_constraint"]})")); - EXPECT_TRUE(matches(plain, modifiers + "staterror", - R"({"constraints": ["gamma_c1_0_constraint", "gamma_c1_1_constraint"]})")); + EXPECT_TRUE(matches(plain, staterror, R"({"constraints": ["gamma_c1_0_constraint", "gamma_c1_1_constraint"]})")); // From the workspace with imported arrays, the components are written on their own as well. Imported ws2{exported}; Doc plainFromArrays{ws2, {exportArrayDist(false)}}; EXPECT_THAT(plainFromArrays.json(), Not(HasSubstr("array_dist"))); - EXPECT_TRUE(matches(plainFromArrays, modifiers + "staterror", + EXPECT_TRUE(matches(plainFromArrays, staterror, R"({"constraints": ["nom_gamma_c1_0_constraint", "nom_gamma_c1_1_constraint"]})")); } @@ -3283,7 +3365,8 @@ TEST(RooFitHS3, TutorialRF515RoundTrip) // The top-level pdf is the RooSimultaneous "main" of the analysis. EXPECT_TRUE(matches(exported, "distributions/main_gaussian_constraints", R"({"type": "array_dist"})")); EXPECT_EQ(exported["distributions/main_gaussian_constraints/components"].num_children(), 3u); - EXPECT_TRUE(matches(exported, "distributions/model_channel1/samples/background1/modifiers/staterror", + // The name of the modifier in the document is kept. + EXPECT_TRUE(matches(exported, "distributions/model_channel1/samples/background1/modifiers/mcstat", R"({"constraints": ["gamma_bin_0_constraint", "gamma_bin_1_constraint"]})")); EXPECT_EQ(reexport(exported), exported.json()); @@ -3497,9 +3580,10 @@ TEST(RooFitHS3, HistFactoryShapeFactor) const std::string json = makeJSON("0.3", freeModifier + "," + constrainedModifier, gammaConstraint("g_c_0")); Imported ws{json}; - // The functions are named as in HistFactory: ShapeFactor for free, ShapeSys for constrained parameters. - EXPECT_NE(ws.function("channel0_free_shapeFactor"), nullptr); - EXPECT_NE(ws.function("channel0_constr_ShapeSys"), nullptr); + // As in HistFactory: ShapeFactor for free, ShapeSys for constrained parameters. + using BinFactorType = RooHistFactorySample::BinFactorType; + EXPECT_EQ(binFactorType(*sigSample(ws), "free"), BinFactorType::ShapeFactor); + EXPECT_EQ(binFactorType(*sigSample(ws), "constr"), BinFactorType::ShapeSys); EXPECT_EQ(countGammaConstraints(ws), 1u); ASSERT_NE(ws.var("g_free_0"), nullptr); EXPECT_FALSE(ws.var("g_free_0")->isConstant()); @@ -3523,13 +3607,13 @@ TEST(RooFitHS3, HistFactoryShapeFactor) Imported ws4{makeJSON("0.3", R"({"type": "shapefactor", "name": "free", "parameters": ["g_free_0", "g_free_1"], "data": {"vals": [0.1, 0.2]}})")}; EXPECT_THAT(ws4.warnings, HasSubstr(R"(the "data.vals" of modifier 'free' of sample 'sig' are ignored)")); - EXPECT_NE(ws4.function("channel0_free_shapeFactor"), nullptr); + EXPECT_EQ(binFactorType(*sigSample(ws4), "free"), BinFactorType::ShapeFactor); EXPECT_EQ(countGammaConstraints(ws4), 0u); // Before HS3 0.3, a "shapesys" with "data.vals" is constrained as before, without deprecation warning. Imported ws5{makeJSON("0.2", R"({"type": "shapesys", "name": "constr", "data": {"vals": [0.1, 0.2]}})")}; EXPECT_THAT(ws5.warnings, Not(HasSubstr("deprecated"))); - EXPECT_NE(ws5.function("channel0_constr_ShapeSys"), nullptr); + EXPECT_EQ(binFactorType(*sigSample(ws5), "constr"), BinFactorType::ShapeSys); EXPECT_EQ(countGammaConstraints(ws5), 2u); } @@ -3540,11 +3624,12 @@ TEST(RooFitHS3, HistFactoryShapeFactorWithoutParameters) RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); Imported ws{HistFactoryDoc{"0.3", R"({"type": "shapefactor", "name": "free", "data": {"vals": [0.1, 0.2]}})"}}; - auto *phf = object(ws, "channel0_free_shapeFactor"); - EXPECT_EQ(phf->paramList().size(), 2u); + EXPECT_EQ(binFactorType(*sigSample(ws), "free"), RooHistFactorySample::BinFactorType::ShapeFactor); + const RooArgList params = binFactorParams(*sigSample(ws), "free"); + EXPECT_EQ(params.size(), 2u); for (const char *name : {"gamma_free_bin_0", "gamma_free_bin_1"}) { auto *gamma = object(ws, name); - EXPECT_NE(phf->paramList().find(*gamma), nullptr) << name; + EXPECT_NE(params.find(*gamma), nullptr) << name; EXPECT_FALSE(gamma->isConstant()) << name; } } @@ -3733,8 +3818,10 @@ TEST(RooFitHS3, HistFactoryShapeFactorSharedBetweenChannels) { Imported ws{ gammasSharedBetweenChannels(R"("type": "shapefactor", "name": "shape")", R"({"contents": [10.0, 20.0]})")}; - EXPECT_NE(ws.function("A_shape_ShapeSys"), nullptr); - EXPECT_NE(ws.function("B_shape_shapeFactor"), nullptr); + EXPECT_EQ(binFactorType(*object(ws, "model_A_bkg_shapes"), "shape"), + RooHistFactorySample::BinFactorType::ShapeSys); + EXPECT_EQ(binFactorType(*object(ws, "model_B_bkg_shapes"), "shape"), + RooHistFactorySample::BinFactorType::ShapeFactor); Doc exported{ws}; EXPECT_TRUE(exported.has("distributions/model_A/samples/0/modifiers/0/constraints")); @@ -3787,7 +3874,7 @@ TEST(RooFitHS3, HistFactoryShapeFactorWithoutName) "constraints": ["g0_constraint", null]})"} .distributions(gammaConstraint("g0")) .sections(defaultDomain({axes({"g0", "g1"}, 0, 5)}))}; - EXPECT_NE(ws.function("channel0_channel0_sig_syst_0_ShapeSys"), nullptr); + EXPECT_EQ(binFactorType(*sigSample(ws), "channel0_sig_syst_0"), RooHistFactorySample::BinFactorType::ShapeSys); } namespace { @@ -3813,8 +3900,7 @@ TEST(RooFitHS3, HistFactoryFunctionGammaWithConstraint) Imported ws{ functionGammaDoc(R"({"type": "staterror", "parameters": ["g0", "g1"], "constraints": ["g0_constraint", null]})") .distributions(gammaConstraint("g0"))}; - auto *phf = object(ws, "mc_stat_channel0"); - RooAbsArg *g0 = phf->paramList().find("g0"); + RooAbsArg *g0 = sigSample(ws)->parameters().find("g0"); ASSERT_NE(g0, nullptr); EXPECT_EQ(dynamic_cast(g0), nullptr); EXPECT_DOUBLE_EQ(static_cast(g0)->getVal(), 1.0); @@ -3880,8 +3966,7 @@ TEST(RooFitHS3, HistFactoryFunctionGammaWithLegacyConstraint) RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); Imported ws{functionGammaDoc(R"({"type": "staterror", "parameters": ["g0", "g1"], "constraint": "Poisson"})")}; - auto *phf = object(ws, "mc_stat_channel0"); - RooAbsArg *g0 = phf->paramList().find("g0"); + RooAbsArg *g0 = sigSample(ws)->parameters().find("g0"); ASSERT_NE(g0, nullptr); EXPECT_EQ(dynamic_cast(g0), nullptr); for (const char *name : {"g0_constraint", "g1_constraint"}) { @@ -3953,8 +4038,12 @@ TEST(RooFitHS3, HistFactoryUnnamedShapeFactorsOfSamples) "model_channel0", {sample("a", contents, R"({"type": "shapefactor", "parameters": ["a0", "a1"]})"), sample("b", contents, R"({"type": "shapefactor", "parameters": ["b0", "b1"]})")})}), defaultDomain({axes({"a0", "a1", "b0", "b1"}, 0, 5)})})}; - EXPECT_NE(ws.function("channel0_channel0_a_syst_0_shapeFactor"), nullptr); - EXPECT_NE(ws.function("channel0_channel0_b_syst_0_shapeFactor"), nullptr); + for (const char *sample : {"a", "b"}) { + auto *hfSample = object(ws, std::string("model_channel0_") + sample + "_shapes"); + EXPECT_EQ(binFactorType(*hfSample, std::string("channel0_") + sample + "_syst_0"), + RooHistFactorySample::BinFactorType::ShapeFactor) + << sample; + } RooAbsPdf *model = ws.pdf("model_channel0"); for (const char *name : {"a0", "a1", "b0", "b1"}) { ASSERT_NE(ws.var(name), nullptr) << name; @@ -4245,6 +4334,10 @@ TEST(RooFitHS3, HistFactoryImportEnablesBinnedLikelihood) // HistFactory exporter split it at the "*" into the factors "1 + 0.2" and "k", which changes the model. TEST(RooFitHS3, FormulaWithTopLevelSumIsNotSplit) { + // The formulas are decomposed for the RooFit objects of the HistFactory models before ROOT 6.44, which the test + // compares with the ones that are read back. + ScopedConfig config{{useHistFactoryPdf(false)}}; + for (std::string expression : {"1 + 0.2 * k", "1.5 - k * 0.2", "2e-1 * k + 1"}) { SCOPED_TRACE(expression); RooWorkspace ws; @@ -4265,10 +4358,12 @@ TEST(RooFitHS3, FormulaWithTopLevelSumIsNotSplit) } // A formula among the factors of a sample is written as one modifier per factor, unless that makes the channel -// invalid: then it stays one custom modifier. +// invalid: then it stays one custom modifier. This is about the RooFit objects of the HistFactory models before ROOT +// 6.44: a RooHistFactoryPdf keeps its custom modifiers as they are. TEST(RooFitHS3, FormulaIsNotSplitIfThatMakesTheChannelInvalid) { RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::ERROR); + ScopedConfig config{{useHistFactoryPdf(false)}}; const std::string b2 = R"({"name": "b2", "type": "generic_function", "expression": "b*b"})"; const std::string bc = R"({"name": "bc", "type": "generic_function", "expression": "b*c"})"; @@ -4304,6 +4399,10 @@ TEST(RooFitHS3, FormulaIsNotSplitIfThatMakesTheChannelInvalid) // formula is linear and the normsys that it is written as interpolates exponentially. TEST(RooFitHS3, FormulaNormSysVariations) { + // The formulas are decomposed for the RooFit objects of the HistFactory models before ROOT 6.44, which the test + // compares with the ones that are read back. + ScopedConfig config{{useHistFactoryPdf(false)}}; + RooWorkspace ws; importFormulaNormSysChannel(ws); // The import enables the binned likelihood, which the first export doesn't have. @@ -4938,7 +5037,11 @@ TEST(RooFitHS3, HistFactoryExportDoesNotDependOnNames) meas.AddChannel(cr); meas.CollectHistograms(); - std::unique_ptr ws{MakeModelAndMeasurementFast(meas)}; + // The test modifies the RooFit objects of the HistFactory models before ROOT 6.44, which the exporter has to + // recognize by their structure. + HistoToWorkspaceFactoryFast::Configuration cfg; + cfg.useHistFactoryPdf = false; + std::unique_ptr ws{MakeModelAndMeasurementFast(meas, cfg)}; ASSERT_NE(ws, nullptr); auto *mc = static_cast(ws->obj("ModelConfig")); RooAbsData *data = ws->data("obsData"); @@ -5649,14 +5752,14 @@ TEST(RooFitHS3, HistFactoryMultiDimensionalBinOrder) }; Imported ws{makeJSON("0.3")}; - auto *phf = object(ws, "mc_stat_channel0"); + RooHistFactorySample *sample = sigSample(ws); for (int ix = 0; ix < 2; ++ix) { for (int iy = 0; iy < 3; ++iy) { // The constraint of each gamma has the relative uncertainty of its bin. auto *sigma = dynamic_cast(ws.arg((gammaName(ix, iy) + "_sigma").c_str())); ASSERT_NE(sigma, nullptr) << gammaName(ix, iy); EXPECT_NEAR(sigma->getVal(), relError(ix, iy), 1e-12) << gammaName(ix, iy); - // In its bin, the ParamHistFunc takes the value of that gamma. + // In its bin, the sample is scaled with that gamma. ws.var(gammaName(ix, iy))->setVal(1.0 + 0.001 * (1 + 3 * ix + iy)); } } @@ -5664,7 +5767,9 @@ TEST(RooFitHS3, HistFactoryMultiDimensionalBinOrder) for (int iy = 0; iy < 3; ++iy) { ws.var("x")->setBin(ix); ws.var("y")->setBin(iy); - EXPECT_DOUBLE_EQ(phf->getVal(), ws.var(gammaName(ix, iy))->getVal()) << "bin (" << ix << ", " << iy << ")"; + // The value of the sample is a density, and the bins have unit volume. + EXPECT_DOUBLE_EQ(sample->getVal(), content(ix, iy) * ws.var(gammaName(ix, iy))->getVal()) + << "bin (" << ix << ", " << iy << ")"; } } for (int ix = 0; ix < 2; ++ix) { @@ -5676,7 +5781,8 @@ TEST(RooFitHS3, HistFactoryMultiDimensionalBinOrder) // The export writes the gammas in the same order. Doc exported{ws}; EXPECT_TRUE(matches(exported, "distributions/model_channel0", R"({"axes": [{"name": "x"}, {"name": "y"}]})")); - EXPECT_TRUE(matches(exported, "distributions/model_channel0/samples/sig/modifiers/staterror", + // The staterror modifier, which has no name in the document, keeps the name that the import gave it. + EXPECT_TRUE(matches(exported, "distributions/model_channel0/samples/sig/modifiers/channel0_sig_syst_0", R"({"parameters": ["g_0_0", "g_0_1", "g_0_2", "g_1_0", "g_1_1", "g_1_2"]})")); // Documents before HS3 0.3 don't define the order; they are rejected. @@ -7085,7 +7191,8 @@ TEST(RooFitHS3, HistFactoryConstraintSharedWithOtherPdf) Doc exported{ws, {exportArrayDist(arrays)}}; const std::string modifiers = "distributions/" + hfName + "/samples/signal/modifiers/"; EXPECT_TRUE(matches(exported, "distributions/shared_constraint", R"({"type": "gaussian_dist"})")); - EXPECT_TRUE(matches(exported, modifiers + "theta", R"({"constraints": ["shared_constraint"]})")); + // The modifiers keep their names in the document. + EXPECT_TRUE(matches(exported, modifiers + "shared", R"({"constraints": ["shared_constraint"]})")); // The product only keeps the factor that is not a term of the likelihood. EXPECT_TRUE(matches(exported, "distributions/" + otherName, R"({"factors": ["shape"]})")); // The constraint that only the HistFactory channel uses is still collected. @@ -7094,7 +7201,7 @@ TEST(RooFitHS3, HistFactoryConstraintSharedWithOtherPdf) EXPECT_EQ(exported.has("distributions/" + array), arrays); if (arrays) { EXPECT_TRUE(matches(exported, "distributions/" + array, R"({"components": [{"x": "eta_obs"}]})")); - EXPECT_TRUE(matches(exported, modifiers + "eta", R"({"constraints": [")" + array + R"("]})")); + EXPECT_TRUE(matches(exported, modifiers + "own", R"({"constraints": [")" + array + R"("]})")); } // Valid HS3 0.3: read back without warnings. @@ -7188,7 +7295,7 @@ TEST(RooFitHS3, HistFactoryConstraintAlsoExternal) Doc exported{ws}; EXPECT_TRUE(matches(exported, "distributions/constraint", R"({"type": "gaussian_dist"})")); EXPECT_TRUE( - matches(exported, "distributions/HF/samples/signal/modifiers/theta", R"({"constraints": ["constraint"]})")); + matches(exported, "distributions/HF/samples/signal/modifiers/syst", R"({"constraints": ["constraint"]})")); EXPECT_TRUE(matches(exported, "likelihoods/0", R"({"aux_distributions": ["constraint"]})")); } diff --git a/roofit/man/man1/hist2workspace.1 b/roofit/man/man1/hist2workspace.1 index 2bcee0e5aa4cf..aa603a7cd8f31 100644 --- a/roofit/man/man1/hist2workspace.1 +++ b/roofit/man/man1/hist2workspace.1 @@ -1,6 +1,6 @@ .TH hist2workspace 1 .SH SYNOPSIS -usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] +usage: hist2workspace [-h] [-v] [-vv] [-disable_binned_fit_optimization] [-legacy_model_structure] .SH DESCRIPTION hist2workspace is a utility to create RooFit/RooStats workspace from histograms @@ -13,3 +13,5 @@ switch HistFactory message stream to INFO level switch HistFactory message stream to DEBUG level .IP -disable_binned_fit_optimization disable the binned fit optimization used in HistFactory since ROOT 6.28 +.IP -legacy_model_structure +build the channels from several RooFit objects per histogram, as before ROOT 6.44 From 12c2aa9fd632b0328b99c6baa25430b9b8b467e0 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 16 Nov 2026 18:04:59 +0000 Subject: [PATCH 3/5] [RF] Let objects in a RooWorkspace store payloads next to it RooWorkspace::Write() and the reading in RooWorkspace::Streamer() open a WorkspaceStreamingContext, in which the streamers of the workspace objects can share state. With it, they can write large payloads to the file after the workspace, which is limited to 1 GB like all objects in ROOT files. Assisted-by: claude-code:claude-opus-5-5 --- roofit/roofitcore/CMakeLists.txt | 2 + .../RooFit/Detail/WorkspaceStreamingContext.h | 69 ++++++++++++++++++ roofit/roofitcore/inc/RooWorkspace.h | 3 + roofit/roofitcore/src/RooWorkspace.cxx | 28 ++++++++ .../src/WorkspaceStreamingContext.cxx | 72 +++++++++++++++++++ 5 files changed, 174 insertions(+) create mode 100644 roofit/roofitcore/inc/RooFit/Detail/WorkspaceStreamingContext.h create mode 100644 roofit/roofitcore/src/WorkspaceStreamingContext.cxx diff --git a/roofit/roofitcore/CMakeLists.txt b/roofit/roofitcore/CMakeLists.txt index af0f2340177b3..e1f917a151e3b 100644 --- a/roofit/roofitcore/CMakeLists.txt +++ b/roofit/roofitcore/CMakeLists.txt @@ -121,6 +121,7 @@ set(ROOFITCORE_HEADERS RooFit/Detail/NormalizationHelpers.h RooFit/Detail/RooNLLVarNew.h RooFit/Detail/RooNormalizedPdf.h + RooFit/Detail/WorkspaceStreamingContext.h RooFit/EvalContext.h RooFit/Evaluator.h RooFit/Floats.h @@ -443,6 +444,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(RooFitCore src/TestStatistics/RooUnbinnedL.cxx src/TestStatistics/buildLikelihood.cxx src/TestStatistics/SharedOffset.cxx + src/WorkspaceStreamingContext.cxx ${RooFitMPTestStatisticsSources} DICTIONARY_OPTIONS "-writeEmptyRootPCM" diff --git a/roofit/roofitcore/inc/RooFit/Detail/WorkspaceStreamingContext.h b/roofit/roofitcore/inc/RooFit/Detail/WorkspaceStreamingContext.h new file mode 100644 index 0000000000000..97a7d1d164896 --- /dev/null +++ b/roofit/roofitcore/inc/RooFit/Detail/WorkspaceStreamingContext.h @@ -0,0 +1,69 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#ifndef RooFit_Detail_WorkspaceStreamingContext_h +#define RooFit_Detail_WorkspaceStreamingContext_h + +#include +#include +#include +#include + +class RooWorkspace; +class TFile; + +namespace RooFit { +namespace Detail { + +/// State shared by the objects that are streamed as part of one RooWorkspace, for as long as the workspace is +/// written or read. Classes with large payloads use it to store them outside of the workspace buffer, which is +/// limited to 1 GB. RooWorkspace::Write() creates the context for writing, and RooWorkspace::Streamer() for reading. +class WorkspaceStreamingContext { +public: + /// State that one class keeps while the workspace is streamed. + class Extension { + public: + virtual ~Extension() = default; + /// Called after all objects of the workspace were streamed. When writing, the workspace is then already in the + /// file. + virtual void finalize() {} + }; + + WorkspaceStreamingContext(RooWorkspace const &workspace, TFile *file); + ~WorkspaceStreamingContext(); + + WorkspaceStreamingContext(WorkspaceStreamingContext const &) = delete; + WorkspaceStreamingContext &operator=(WorkspaceStreamingContext const &) = delete; + + /// The context of the workspace that this thread is streaming, or nullptr. + static WorkspaceStreamingContext *current(); + + RooWorkspace const &workspace() const { return _workspace; } + /// The file that the workspace is written to, if objects can write their payloads to it, else nullptr. + TFile *file() const { return _file; } + + Extension *extension(std::string const &name) const; + Extension &addExtension(std::string const &name, std::unique_ptr extension); + + void finalize(); + +private: + RooWorkspace const &_workspace; + TFile *_file = nullptr; + WorkspaceStreamingContext *_previous = nullptr; + std::vector>> _extensions; +}; + +} // namespace Detail +} // namespace RooFit + +#endif diff --git a/roofit/roofitcore/inc/RooWorkspace.h b/roofit/roofitcore/inc/RooWorkspace.h index 6f84ff97eaac9..d7640ca146cbb 100644 --- a/roofit/roofitcore/inc/RooWorkspace.h +++ b/roofit/roofitcore/inc/RooWorkspace.h @@ -53,6 +53,9 @@ class RooWorkspace : public TNamed { TObject *Clone(const char *newname="") const override; + Int_t Write(const char *name = nullptr, Int_t option = 0, Int_t bufsize = 0) const override; + Int_t Write(const char *name = nullptr, Int_t option = 0, Int_t bufsize = 0) override; + bool importClassCode(const char* pat="*", bool doReplace=false) ; bool importClassCode(TClass* theClass, bool doReplace=false) ; diff --git a/roofit/roofitcore/src/RooWorkspace.cxx b/roofit/roofitcore/src/RooWorkspace.cxx index 899702865e60b..7c5802559f5e4 100644 --- a/roofit/roofitcore/src/RooWorkspace.cxx +++ b/roofit/roofitcore/src/RooWorkspace.cxx @@ -62,6 +62,7 @@ and try reading again. #include #include #include +#include #include "TBuffer.h" #include "TInterpreter.h" @@ -2530,7 +2531,9 @@ void RooWorkspace::Streamer(TBuffer &R__b) { if (R__b.IsReading()) { + RooFit::Detail::WorkspaceStreamingContext context{*this, nullptr}; R__b.ReadClassBuffer(RooWorkspace::Class(), this); + context.finalize(); // Perform any pass-2 schema evolution here for (RooAbsArg *node : _allOwnedNodes) { @@ -2631,6 +2634,31 @@ void RooWorkspace::Streamer(TBuffer &R__b) } } +//////////////////////////////////////////////////////////////////////////////// +/// Write the workspace to the current directory, like TObject::Write(). +/// +/// Objects in the workspace can store large payloads next to the workspace in the file, after the workspace itself +/// was written. HistFactory models do that with their histograms, so that the workspace is not limited by the 1 GB +/// size limit of single objects in ROOT files. This only happens when writing with this function or with +/// writeToFile(), not with TDirectory::WriteObject() or TDirectory::WriteTObject(), which write everything into the +/// workspace object. HistFactory writes an RNTuple, which also writes the other objects in memory that are attached to +/// the file, like TFile::Write(). + +Int_t RooWorkspace::Write(const char *name, Int_t option, Int_t bufsize) const +{ + TFile *file = gDirectory ? gDirectory->GetFile() : nullptr; + const bool canWritePayloads = file && file->IsWritable() && file->IsBinary(); + RooFit::Detail::WorkspaceStreamingContext context{*this, canWritePayloads ? file : nullptr}; + const Int_t nbytes = TObject::Write(name, option, bufsize); + context.finalize(); + return nbytes; +} + +Int_t RooWorkspace::Write(const char *name, Int_t option, Int_t bufsize) +{ + return static_cast(this)->Write(name, option, bufsize); +} + diff --git a/roofit/roofitcore/src/WorkspaceStreamingContext.cxx b/roofit/roofitcore/src/WorkspaceStreamingContext.cxx new file mode 100644 index 0000000000000..55887500edb34 --- /dev/null +++ b/roofit/roofitcore/src/WorkspaceStreamingContext.cxx @@ -0,0 +1,72 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include + +#include + +namespace { + +thread_local RooFit::Detail::WorkspaceStreamingContext *currentContext = nullptr; + +} // namespace + +namespace RooFit { +namespace Detail { + +WorkspaceStreamingContext::WorkspaceStreamingContext(RooWorkspace const &workspace, TFile *file) + : _workspace{workspace}, _file{file}, _previous{currentContext} +{ + currentContext = this; +} + +WorkspaceStreamingContext::~WorkspaceStreamingContext() +{ + currentContext = _previous; +} + +WorkspaceStreamingContext *WorkspaceStreamingContext::current() +{ + return currentContext; +} + +/// The extension with a given name, or nullptr if there is none. +WorkspaceStreamingContext::Extension *WorkspaceStreamingContext::extension(std::string const &name) const +{ + for (auto const &item : _extensions) { + if (item.first == name) { + return item.second.get(); + } + } + return nullptr; +} + +WorkspaceStreamingContext::Extension & +WorkspaceStreamingContext::addExtension(std::string const &name, std::unique_ptr extension) +{ + if (this->extension(name)) { + throw std::invalid_argument("WorkspaceStreamingContext: there is already an extension named \"" + name + "\""); + } + _extensions.emplace_back(name, std::move(extension)); + return *_extensions.back().second; +} + +/// Finalize the extensions in the order in which they were added. +void WorkspaceStreamingContext::finalize() +{ + for (auto const &item : _extensions) { + item.second->finalize(); + } +} + +} // namespace Detail +} // namespace RooFit From 7a68b1d3599ce9c37b445fa242abbbbcf9b51455 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 16 Nov 2026 18:04:59 +0000 Subject: [PATCH 4/5] [RF][HF] Write the data of HistFactory samples to an RNTuple When a workspace is written with RooWorkspace::Write(), the data of all its RooHistFactorySamples goes to the RNTuple `_histfactory_arrays`, committed after the workspace itself. HistFactory workspaces are then no longer limited to 1 GB. hist2workspace writes its workspaces like this. Assisted-by: claude-code:claude-opus-5-5 --- README/ReleaseNotes/v644/index.md | 8 + roofit/histfactory/CMakeLists.txt | 1 + .../src/MakeModelAndMeasurementsFast.cxx | 7 +- .../histfactory/src/RooHistFactorySample.cxx | 188 +++++++++++++++++- .../test/testRooHistFactoryPdf.cxx | 171 ++++++++++++++-- 5 files changed, 348 insertions(+), 27 deletions(-) diff --git a/README/ReleaseNotes/v644/index.md b/README/ReleaseNotes/v644/index.md index a5f7498c21bf3..e888d5bf29a9b 100644 --- a/README/ReleaseNotes/v644/index.md +++ b/README/ReleaseNotes/v644/index.md @@ -74,6 +74,14 @@ navigates the internal structure of HistFactory models, e.g. to retrieve the `RooHistFunc` of a sample, has to be adapted. The modifiers of a sample are available via `RooHistFactorySample::data()`. +When a workspace is written to a ROOT file with `RooWorkspace::Write()` or +`RooWorkspace::writeToFile()`, the histogram data of all its samples goes to an +RNTuple next to the workspace, named after the workspace with the suffix +`_histfactory_arrays`. Like this, HistFactory workspaces are no longer limited +by the 1 GB size limit of single objects in ROOT files. `hist2workspace` writes +its workspaces like this. With `TDirectory::WriteObject()`, the histograms stay +in the workspace object. + The HS3 importer also creates `RooHistFactoryPdf` objects for the `histfactory_dist` distributions, and the exporter writes them with the names of their modifiers. diff --git a/roofit/histfactory/CMakeLists.txt b/roofit/histfactory/CMakeLists.txt index 08f9b99c6c09b..1308f7249107a 100644 --- a/roofit/histfactory/CMakeLists.txt +++ b/roofit/histfactory/CMakeLists.txt @@ -65,6 +65,7 @@ ROOT_STANDARD_LIBRARY_PACKAGE(HistFactory DEPENDENCIES RooFit RooFitCore + ROOTNTuple Tree RIO Hist diff --git a/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx b/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx index 39103c613fbfe..6235346f77e3e 100644 --- a/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx +++ b/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx @@ -183,7 +183,9 @@ RooStats::HistFactory::MakeModelAndMeasurementFast(RooStats::HistFactory::Measur + ch_name + "_" + rowTitle + "_model.root"; cxcoutIHF << "Opening File to hold channel: " << ChannelFileName << std::endl; std::unique_ptr chanFile{TFile::Open( ChannelFileName.c_str(), "RECREATE" )}; - chanFile->WriteTObject(ws_single.get()); + // RooWorkspace::Write() can store the histograms next to the workspace, see RooHistFactorySample. + chanFile->cd(); + ws_single->Write(); // Now, write the measurement to the file // Make a new measurement for only this channel RooStats::HistFactory::Measurement meas_chan( measurement ); @@ -223,7 +225,8 @@ RooStats::HistFactory::MakeModelAndMeasurementFast(RooStats::HistFactory::Measur cxcoutEHF << "Error: Failed to open file " << CombinedFileName << std::endl; throw hf_exc(); } - combFile->WriteTObject(ws.get()); + combFile->cd(); + ws->Write(); cxcoutPHF << "Writing combined measurement to file: " << CombinedFileName << std::endl; measurement.writeToFile( combFile.get() ); } diff --git a/roofit/histfactory/src/RooHistFactorySample.cxx b/roofit/histfactory/src/RooHistFactorySample.cxx index 14bdfdf820f2d..e45c45b214983 100644 --- a/roofit/histfactory/src/RooHistFactorySample.cxx +++ b/roofit/histfactory/src/RooHistFactorySample.cxx @@ -31,6 +31,15 @@ the bin volume. In the binned likelihood, the sample is evaluated as the yield \ In contrast to the HistFactory models before ROOT 6.44, which consist of several RooFit objects for each histogram, a sample is a single object. The histogram data is shared between the copies of a sample. + +When a RooWorkspace is written to a TFile with RooWorkspace::Write() or RooWorkspace::writeToFile(), the histogram +data of all its samples is written to an RNTuple in the top directory of the file, named after the workspace with the +suffix `_histfactory_arrays`, and the workspace refers to its entries. Like this, the workspace is not limited by the +1 GB size limit of single objects in ROOT files. When the RNTuple name is taken, for example because the workspace is +written again, a number is appended. The RNTuple stays in the file when the workspace is deleted from it. As for all +RNTuples written to a TFile, the other objects in memory that are attached to the file are written too. In all other +cases, for example with TDirectory::WriteObject() or when a sample is written without a workspace, the data is written +together with the sample. **/ #include @@ -40,15 +49,28 @@ a sample is a single object. The histogram data is shared between the copies of #include #include #include +#include #include #include #include +#include + +#include +#include +#include +#include +#include +#include #include +#include #include +#include #include +#include #include +#include namespace { @@ -523,16 +545,174 @@ void RooHistFactorySample::printMetaArgs(std::ostream &os) const os << " "; } +namespace { + +using Data = RooHistFactorySample::Data; +using RooFit::Detail::WorkspaceStreamingContext; + +/// How the data of a sample is stored, written after the members of the sample. +enum StorageType : Int_t { + inlineStorage = 0, + rntupleStorage = 1 +}; + +std::unique_ptr makeModel() +{ + auto model = ROOT::RNTupleModel::CreateBare(); + model->MakeField("data"); + return model; +} + +/// Writes the data of the samples in a workspace to one RNTuple. +class DataWriter : public WorkspaceStreamingContext::Extension { +public: + DataWriter(TFile &file, std::string const &workspaceName) + { + const std::string baseName = workspaceName + "_histfactory_arrays"; + _name = baseName; + for (int i = 1; file.GetKey(_name.c_str()); ++i) { + _name = baseName + "_" + std::to_string(i); + } + ROOT::RNTupleWriteOptions options; + options.SetCompression(file.GetCompressionSettings()); + _writer = ROOT::RNTupleWriter::Append(makeModel(), _name, file, options); + _entry = _writer->CreateRawPtrWriteEntry(); + } + + std::string const &name() const { return _name; } + + /// Write the data, if it was not written already, and return its entry number. + std::uint64_t add(Data const &data) + { + auto found = _entries.find(&data); + if (found != _entries.end()) { + return found->second; + } + _entry->BindRawPtr("data", &data); + _writer->Fill(*_entry); + const std::uint64_t index = _entries.size(); + _entries.emplace(&data, index); + return index; + } + + void finalize() override + { + _writer->CommitDataset(); + _writer.reset(); + } + +private: + std::string _name; + std::unique_ptr _writer; + std::unique_ptr _entry; + std::unordered_map _entries; +}; + +/// Reads the data of the samples from the RNTuples of a file. +class DataReader : public WorkspaceStreamingContext::Extension { +public: + std::shared_ptr get(TFile &file, std::string const &name, std::uint64_t index) + { + auto &source = _sources[name]; + auto found = source.data.find(index); + if (found != source.data.end()) { + return found->second; + } + if (!source.reader) { + // The first cycle is the one that was written with the workspace. Later cycles can come from copying + // the file, e.g. with hadd, which writes the workspace and with it a new RNTuple before copying this one. + std::unique_ptr anchor{file.Get((name + ";1").c_str())}; + if (!anchor) { + throw std::runtime_error("RooHistFactorySample: the RNTuple \"" + name + + "\" with the histogram data is missing in the file \"" + file.GetName() + "\""); + } + source.reader = ROOT::RNTupleReader::Open(makeModel(), *anchor); + source.entry = source.reader->GetModel().CreateBareEntry(); + } + auto data = std::make_shared(); + source.entry->BindRawPtr("data", data.get()); + source.reader->LoadEntry(index, *source.entry); + source.data.emplace(index, data); + return data; + } + +private: + struct Source { + std::unique_ptr reader; + std::unique_ptr entry; + std::unordered_map> data; ///< To share data between samples again. + }; + std::map _sources; +}; + +/// The file of the RNTuple for the data, if the buffer writes a workspace to a file with RooWorkspace::Write(). +TFile *fileForRNTuple(WorkspaceStreamingContext const *context, TBuffer &buffer) +{ + if (!context || !context->file() || buffer.GetParent() != context->file()) { + return nullptr; + } + // Not for TMessage or TBufferJSON, for example. + return buffer.IsA() == TBufferFile::Class() ? context->file() : nullptr; +} + +template +T &getExtension(WorkspaceStreamingContext &context, std::string const &name, Create &&create) +{ + if (auto *extension = context.extension(name)) { + return static_cast(*extension); + } + return static_cast(context.addExtension(name, create())); +} + +} // namespace + void RooHistFactorySample::Streamer(TBuffer &buffer) { static TClass *dataClass = TClass::GetClass(); if (buffer.IsReading()) { buffer.ReadClassBuffer(RooHistFactorySample::Class(), this); - auto data = std::make_shared(); - buffer.StreamObject(data.get(), dataClass); - _data = std::move(data); + Int_t storage = inlineStorage; + buffer >> storage; + if (storage == inlineStorage) { + auto data = std::make_shared(); + buffer.StreamObject(data.get(), dataClass); + _data = std::move(data); + return; + } + if (storage != rntupleStorage) { + throw std::runtime_error("RooHistFactorySample::Streamer(): unknown storage of the histogram data"); + } + std::string name; + ULong64_t index = 0; + buffer.ReadStdString(&name); + buffer >> index; + auto *file = dynamic_cast(buffer.GetParent()); + if (!file) { + throw std::runtime_error("RooHistFactorySample::Streamer(): the histogram data is in the RNTuple \"" + name + + "\", but the sample is not read from a file"); + } + if (auto *context = WorkspaceStreamingContext::current()) { + auto &reader = getExtension(*context, "RooHistFactorySample::DataReader", + [] { return std::make_unique(); }); + _data = reader.get(*file, name, index); + } else { + _data = DataReader{}.get(*file, name, index); + } } else { buffer.WriteClassBuffer(RooHistFactorySample::Class(), this); - buffer.StreamObject(const_cast(_data.get()), dataClass); + auto *context = WorkspaceStreamingContext::current(); + TFile *file = fileForRNTuple(context, buffer); + if (!file) { + buffer << static_cast(inlineStorage); + buffer.StreamObject(const_cast(_data.get()), dataClass); + return; + } + auto &writer = getExtension(*context, "RooHistFactorySample::DataWriter", [&] { + return std::make_unique(*file, context->workspace().GetName()); + }); + const ULong64_t index = writer.add(*_data); + buffer << static_cast(rntupleStorage); + buffer.WriteStdString(&writer.name()); + buffer << index; } } diff --git a/roofit/histfactory/test/testRooHistFactoryPdf.cxx b/roofit/histfactory/test/testRooHistFactoryPdf.cxx index afecc7f2b0554..d94361e4f1e07 100644 --- a/roofit/histfactory/test/testRooHistFactoryPdf.cxx +++ b/roofit/histfactory/test/testRooHistFactoryPdf.cxx @@ -12,9 +12,12 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -140,6 +143,48 @@ void setParameters(TRandom3 &rng, RooWorkspace &ws1, RooWorkspace &ws2) } } +/// A workspace with one sample whose data is much larger than the rest of the workspace. +std::unique_ptr makeLargeSampleWorkspace(std::size_t nBins) +{ + RooRealVar x{"x", "x", 0, 1}; + x.setBins(nBins); + RooRealVar mu{"mu", "mu", 1.0, 0, 5}; + RooRealVar alpha{"alpha", "alpha", 0.3, -5, 5}; + TRandom3 rng{5}; + RooHistFactorySample::Spec spec; + spec.sampleName = "sample"; + RooHistFactorySample::HistoSys histoSys{"syst", &alpha, {}, {}}; + for (std::size_t i = 0; i < nBins; ++i) { + spec.contents.push_back(rng.Uniform(1, 2)); + histoSys.lo.push_back(0.9 * spec.contents.back()); + histoSys.hi.push_back(1.2 * spec.contents.back()); + } + spec.histoSys.push_back(histoSys); + spec.normFactors.push_back({"mu", &mu}); + RooHistFactorySample sample{"sample_shapes", "sample_shapes", RooArgList{x}, spec}; + RooHistFactoryPdf pdf{"model", "model", RooArgList{sample}}; + auto ws = std::make_unique("w"); + ws->import(pdf, RooFit::Silence()); + return ws; +} + +void expectSameSample(RooWorkspace &ws, RooWorkspace &wsRead) +{ + auto *sample = dynamic_cast(ws.function("sample_shapes")); + auto *sampleRead = dynamic_cast(wsRead.function("sample_shapes")); + ASSERT_NE(sampleRead, nullptr); + EXPECT_EQ(sampleRead->data().contents, sample->data().contents); + EXPECT_EQ(sampleRead->data().histoSysLo, sample->data().histoSysLo); + EXPECT_EQ(sampleRead->data().histoSysHi, sample->data().histoSysHi); + EXPECT_EQ(sampleRead->data().histoSysNames, sample->data().histoSysNames); + EXPECT_EQ(sampleRead->data().edges, sample->data().edges); + std::vector values(sample->numBins()); + std::vector valuesRead(sample->numBins()); + sample->binValues(values, false); + sampleRead->binValues(valuesRead, false); + EXPECT_EQ(valuesRead, values); +} + } // namespace class RooHistFactoryPdfTest : public testing::TestWithParam {}; @@ -228,7 +273,8 @@ TEST_P(RooHistFactoryPdfTest, Codegen) } #endif -/// The samples and their data survive the writing to a file, and the data is shared between clones. +/// The samples and their data survive the writing to a file, with the data in the workspace object or in an RNTuple, +/// and the data is shared between clones. TEST_P(RooHistFactoryPdfTest, PersistenceAndClones) { RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); @@ -239,30 +285,39 @@ TEST_P(RooHistFactoryPdfTest, PersistenceAndClones) { TFile file(fileName.c_str(), "RECREATE"); file.WriteObject(ws.get(), "ws"); + ws->Write("wsRNTuple"); } std::unique_ptr file{TFile::Open(fileName.c_str())}; - std::unique_ptr wsRead{file->Get("ws")}; - ASSERT_NE(wsRead, nullptr); + // Only RooWorkspace::Write() puts the data of the samples into an RNTuple. + TKey *key = file->GetKey("combined_histfactory_arrays"); + ASSERT_NE(key, nullptr); + EXPECT_STREQ(key->GetClassName(), "ROOT::RNTuple"); + EXPECT_EQ(file->GetKey("combined_histfactory_arrays_1"), nullptr); auto *pdf = dynamic_cast(ws->pdf("ch_model")); - auto *pdfRead = dynamic_cast(wsRead->pdf("ch_model")); - ASSERT_NE(pdfRead, nullptr); - ASSERT_EQ(pdfRead->numSamples(), pdf->numSamples()); - - TRandom3 rng{3}; - setParameters(rng, *ws, *wsRead); - for (std::size_t i = 0; i < pdf->numSamples(); ++i) { - RooHistFactorySample const &sample = pdf->sample(i); - RooHistFactorySample const &sampleRead = pdfRead->sample(i); - EXPECT_EQ(sampleRead.sampleName(), sample.sampleName()); - EXPECT_EQ(sampleRead.data().contents, sample.data().contents); - EXPECT_EQ(sampleRead.data().histoSysHi, sample.data().histoSysHi); - EXPECT_EQ(sampleRead.data().edges, sample.data().edges); - std::vector values(sample.numBins()); - std::vector valuesRead(sample.numBins()); - sample.binValues(values, false); - sampleRead.binValues(valuesRead, false); - EXPECT_EQ(valuesRead, values) << sample.sampleName(); + for (const char *wsName : {"ws", "wsRNTuple"}) { + std::unique_ptr wsRead{file->Get(wsName)}; + ASSERT_NE(wsRead, nullptr) << wsName; + auto *pdfRead = dynamic_cast(wsRead->pdf("ch_model")); + ASSERT_NE(pdfRead, nullptr); + ASSERT_EQ(pdfRead->numSamples(), pdf->numSamples()); + + TRandom3 rng{3}; + setParameters(rng, *ws, *wsRead); + for (std::size_t i = 0; i < pdf->numSamples(); ++i) { + RooHistFactorySample const &sample = pdf->sample(i); + RooHistFactorySample const &sampleRead = pdfRead->sample(i); + EXPECT_EQ(sampleRead.sampleName(), sample.sampleName()); + EXPECT_EQ(sampleRead.data().contents, sample.data().contents); + EXPECT_EQ(sampleRead.data().histoSysHi, sample.data().histoSysHi); + EXPECT_EQ(sampleRead.data().edges, sample.data().edges); + std::vector values(sample.numBins()); + std::vector valuesRead(sample.numBins()); + sample.binValues(values, false); + sampleRead.binValues(valuesRead, false); + EXPECT_EQ(valuesRead, values) << wsName << " " << sample.sampleName(); + } + ws->loadSnapshot("NominalParamValues"); } // A clone shares the data, until its interpolation codes are changed. @@ -275,6 +330,80 @@ TEST_P(RooHistFactoryPdfTest, PersistenceAndClones) EXPECT_EQ(signal.data().histoSysCodes, (std::vector{4, 4})); } +/// With RooWorkspace::Write(), the sample data is not part of the workspace object in the file, so that the workspace +/// is not limited by the size limit of objects in ROOT files. Every time the workspace is written, it takes a new +/// RNTuple. +TEST(RooHistFactoryPdf, DataInRNTuple) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + const std::size_t nBins = 100000; + auto ws = makeLargeSampleWorkspace(nBins); + const std::string fileName = "testRooHistFactoryPdf_rntuple.root"; + { + TFile file(fileName.c_str(), "RECREATE"); + ws->Write(); + ws->Write("w2"); + } + TFile file(fileName.c_str()); + for (const char *name : {"w_histfactory_arrays", "w_histfactory_arrays_1"}) { + TKey *key = file.GetKey(name); + ASSERT_NE(key, nullptr) << name; + EXPECT_STREQ(key->GetClassName(), "ROOT::RNTuple"); + } + EXPECT_LT(file.GetKey("w")->GetObjlen(), nBins * sizeof(double)); + for (const char *name : {"w", "w2"}) { + std::unique_ptr wsRead{file.Get(name)}; + ASSERT_NE(wsRead, nullptr) << name; + expectSameSample(*ws, *wsRead); + } + + const std::string fileName2 = "testRooHistFactoryPdf_rntuple_2.root"; + ASSERT_TRUE(ws->writeToFile(fileName2.c_str())); + TFile file2(fileName2.c_str()); + ASSERT_NE(file2.GetKey("w_histfactory_arrays"), nullptr); + std::unique_ptr wsRead{file2.Get("w")}; + ASSERT_NE(wsRead, nullptr); + expectSameSample(*ws, *wsRead); +} + +/// Files in memory get the RNTuple too. +TEST(RooHistFactoryPdf, DataInRNTupleInMemoryFile) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto ws = makeLargeSampleWorkspace(1000); + TMemFile file("testRooHistFactoryPdf_memfile.root", "RECREATE"); + ws->Write(); + ASSERT_NE(file.GetKey("w_histfactory_arrays"), nullptr); + std::unique_ptr wsRead{file.Get("w")}; + ASSERT_NE(wsRead, nullptr); + expectSameSample(*ws, *wsRead); +} + +/// Without RooWorkspace::Write(), the data is written with the sample. +TEST(RooHistFactoryPdf, DataWithoutRNTuple) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + auto ws = makeLargeSampleWorkspace(1000); + + TBufferFile buffer{TBuffer::kWrite}; + buffer.WriteObject(ws.get()); + buffer.SetReadMode(); + buffer.SetBufferOffset(0); + std::unique_ptr wsRead{static_cast(buffer.ReadObject(RooWorkspace::Class()))}; + ASSERT_NE(wsRead, nullptr); + expectSameSample(*ws, *wsRead); + + TMemFile file("testRooHistFactoryPdf_inline.root", "RECREATE"); + file.WriteObject(ws.get(), "w"); + file.WriteObject(ws->function("sample_shapes"), "sample"); + for (TObject *key : *file.GetListOfKeys()) { + EXPECT_STRNE(static_cast(key)->GetClassName(), "ROOT::RNTuple"); + } + std::unique_ptr wsFromFile{file.Get("w")}; + ASSERT_NE(wsFromFile, nullptr); + expectSameSample(*ws, *wsFromFile); +} + /// Integrals over ranges that don't align with the bins take the fraction of the bins in the range. TEST(RooHistFactoryPdf, RangedIntegral) { From c361943a662d994618de09f31c0a0f04f7013969 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sat, 21 Nov 2026 15:24:51 +0100 Subject: [PATCH 5/5] [RF][HF] Add a configuration flag to not write the Measurement With HistoToWorkspaceFactoryFast::Configuration::writeMeasurement set to false, MakeModelAndMeasurementFast() writes only the workspaces to the files of the channels and of the combined model, without the Measurement and the directories with copies of its histograms. The Measurement holds all histograms, so it is subject to the 1 GB limit of single objects in ROOT files like the workspace was before its histograms could go to an RNTuple, and the workspaces don't need it. Assisted-by: claude-code:claude-opus-5-5 --- README/ReleaseNotes/v644/index.md | 5 ++ .../HistFactory/HistoToWorkspaceFactoryFast.h | 4 ++ .../src/MakeModelAndMeasurementsFast.cxx | 24 +++++---- roofit/histfactory/test/testHistFactory.cxx | 54 +++++++++++++++++++ 4 files changed, 77 insertions(+), 10 deletions(-) diff --git a/README/ReleaseNotes/v644/index.md b/README/ReleaseNotes/v644/index.md index e888d5bf29a9b..34228af455f65 100644 --- a/README/ReleaseNotes/v644/index.md +++ b/README/ReleaseNotes/v644/index.md @@ -82,6 +82,11 @@ by the 1 GB size limit of single objects in ROOT files. `hist2workspace` writes its workspaces like this. With `TDirectory::WriteObject()`, the histograms stay in the workspace object. +The `Measurement` that HistFactory writes next to the workspaces holds all +histograms as well, so it is subject to the same limit. Since the workspaces +don't need it, writing it can be disabled with the new `writeMeasurement` +member of `RooStats::HistFactory::HistoToWorkspaceFactoryFast::Configuration`. + The HS3 importer also creates `RooHistFactoryPdf` objects for the `histfactory_dist` distributions, and the exporter writes them with the names of their modifiers. diff --git a/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h b/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h index 807b65e6054fe..5238a07be886b 100644 --- a/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h +++ b/roofit/histfactory/inc/RooStats/HistFactory/HistoToWorkspaceFactoryFast.h @@ -56,6 +56,10 @@ class HistoToWorkspaceFactoryFast : public TObject { /// \brief Build each channel as a RooHistFactoryPdf of RooHistFactorySample objects (default `true`). If /// `false`, the channels are built as before ROOT 6.44, with several RooFit objects for each histogram. bool useHistFactoryPdf = true; + /// \brief Control whether the Measurement is written to the ROOT files next to the workspaces (default `true`), + /// together with copies of its histograms. The workspaces don't need it. Like any object in a ROOT file, the + /// Measurement, which holds all histograms, is limited to 1 GB. + bool writeMeasurement = true; }; HistoToWorkspaceFactoryFast() {} diff --git a/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx b/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx index 6235346f77e3e..047ebfa2dfef1 100644 --- a/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx +++ b/roofit/histfactory/src/MakeModelAndMeasurementsFast.cxx @@ -186,14 +186,16 @@ RooStats::HistFactory::MakeModelAndMeasurementFast(RooStats::HistFactory::Measur // RooWorkspace::Write() can store the histograms next to the workspace, see RooHistFactorySample. chanFile->cd(); ws_single->Write(); - // Now, write the measurement to the file - // Make a new measurement for only this channel - RooStats::HistFactory::Measurement meas_chan( measurement ); - meas_chan.GetChannels().clear(); - meas_chan.GetChannels().push_back( channel ); - cxcoutIHF << "About to write channel measurement to file" << std::endl; - meas_chan.writeToFile( chanFile.get() ); - cxcoutPHF << "Successfully wrote channel to file" << std::endl; + if (cfg.writeMeasurement) { + // Now, write the measurement to the file + // Make a new measurement for only this channel + RooStats::HistFactory::Measurement meas_chan( measurement ); + meas_chan.GetChannels().clear(); + meas_chan.GetChannels().push_back( channel ); + cxcoutIHF << "About to write channel measurement to file" << std::endl; + meas_chan.writeToFile( chanFile.get() ); + cxcoutPHF << "Successfully wrote channel to file" << std::endl; + } } channel_workspaces.emplace_back(std::move(ws_single)); @@ -227,8 +229,10 @@ RooStats::HistFactory::MakeModelAndMeasurementFast(RooStats::HistFactory::Measur } combFile->cd(); ws->Write(); - cxcoutPHF << "Writing combined measurement to file: " << CombinedFileName << std::endl; - measurement.writeToFile( combFile.get() ); + if (cfg.writeMeasurement) { + cxcoutPHF << "Writing combined measurement to file: " << CombinedFileName << std::endl; + measurement.writeToFile( combFile.get() ); + } } return RooFit::makeOwningPtr(std::move(ws)); diff --git a/roofit/histfactory/test/testHistFactory.cxx b/roofit/histfactory/test/testHistFactory.cxx index 0b1846f263be2..8abe8e1008138 100644 --- a/roofit/histfactory/test/testHistFactory.cxx +++ b/roofit/histfactory/test/testHistFactory.cxx @@ -761,3 +761,57 @@ TEST(HistFactory, ShapeFactorValueAndRange) } } } + +// With HistoToWorkspaceFactoryFast::Configuration::writeMeasurement, the files of the channels and of the combined +// model have the Measurement and the directories with its histograms next to the workspace; without, only the +// workspace. +TEST(HistFactory, WriteMeasurement) +{ + using namespace RooStats::HistFactory; + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + const std::string inputFileName = "TestWriteMeasurement_input.root"; + { + TFile f(inputFileName.c_str(), "RECREATE"); + TH1D data{"data", "data", 2, 1, 2}; + TH1D signal{"signal", "signal", 2, 1, 2}; + data.SetBinContent(1, 12); + data.SetBinContent(2, 21); + signal.SetBinContent(1, 10); + signal.SetBinContent(2, 20); + f.WriteTObject(&data); + f.WriteTObject(&signal); + } + + for (bool writeMeasurement : {true, false}) { + SCOPED_TRACE(writeMeasurement ? "with Measurement" : "without Measurement"); + const std::string prefix = std::string("TestWriteMeasurement_") + (writeMeasurement ? "with" : "without"); + + Measurement meas("meas", "meas"); + meas.SetOutputFilePrefix(prefix); + meas.SetPOI("mu"); + meas.SetLumi(1.0); + meas.SetLumiRelErr(0.1); + Channel chan("channel1"); + chan.SetData("data", inputFileName); + Sample sig("signal", "signal", inputFileName); + sig.AddNormFactor("mu", 1, 0, 3); + chan.AddSample(sig); + meas.AddChannel(chan); + meas.CollectHistograms(); + + HistoToWorkspaceFactoryFast::Configuration cfg; + cfg.writeMeasurement = writeMeasurement; + ASSERT_NE(std::unique_ptr{MakeModelAndMeasurementFast(meas, cfg)}, nullptr); + + for (auto const &[fileSuffix, wsName] : std::vector>{ + {"_channel1_meas_model.root", "channel1"}, {"_combined_meas_model.root", "combined"}}) { + SCOPED_TRACE(fileSuffix); + std::unique_ptr file{TFile::Open((prefix + fileSuffix).c_str())}; + ASSERT_TRUE(file && file->IsOpen()); + EXPECT_NE(std::unique_ptr{file->Get(wsName.c_str())}, nullptr); + EXPECT_EQ(std::unique_ptr{file->Get("meas")} != nullptr, writeMeasurement); + EXPECT_EQ(file->GetKey("channel1_hists") != nullptr, writeMeasurement); + } + } +}