From aba52e2a9ffc2f5e8d1866617dab54f96e524de1 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 13 Nov 2026 13:07:05 +0000 Subject: [PATCH 1/9] [RF][HS3] Enable `bugprone-argument-comment` clang-tidy check Fix the argument comments that don't match the parameter names. Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit 097df953a5492162446fc4c716504b7f3070b330) --- roofit/hs3/test/testRooFitHS3.cxx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roofit/hs3/test/testRooFitHS3.cxx b/roofit/hs3/test/testRooFitHS3.cxx index 1a1acb819c100..9237906def9ea 100644 --- a/roofit/hs3/test/testRooFitHS3.cxx +++ b/roofit/hs3/test/testRooFitHS3.cxx @@ -2537,7 +2537,8 @@ TEST(RooFitHS3, HistFactoryDuplicateHistoSysWithDifferentBinningFails) y.setBins(3); RooWorkspace ws{"ws_incompatible_histosys_binning"}; - importDuplicateHistoSysModel(ws, /*interpCode=*/4, /*low2=*/{7.0, 8.0, 9.0}, /*high2=*/{13.0, 14.0, 15.0}, &y); + importDuplicateHistoSysModel(ws, /*interpCode=*/4, /*low2Contents=*/{7.0, 8.0, 9.0}, + /*high2Contents=*/{13.0, 14.0, 15.0}, &y); expectExportThrowsWithError(ws, "histogram binning differs"); } @@ -2545,7 +2546,7 @@ TEST(RooFitHS3, HistFactoryDuplicateHistoSysWithDifferentBinningFails) TEST(RooFitHS3, HistFactoryDuplicateHistoSysWithNonDefaultInterpolationFails) { RooWorkspace ws{"ws_incompatible_histosys_interpolation"}; - importDuplicateHistoSysModel(ws, /*interpCode=*/2, /*low2=*/{9.0, 16.0}, /*high2=*/{11.0, 24.0}); + importDuplicateHistoSysModel(ws, /*interpCode=*/2, /*low2Contents=*/{9.0, 16.0}, /*high2Contents=*/{11.0, 24.0}); expectExportThrowsWithError(ws, "this interpolation cannot currently be combined for duplicate histosys modifiers"); } From 2bcc12ca787761332e47f164a012e62620c20c37 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 13 Nov 2026 13:29:42 +0000 Subject: [PATCH 2/9] [RF][HS3] Enable `cppcoreguidelines-pro-type-member-init` check Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit 3c759b4cb786dd87d754ee69c33d453a492fad95) --- roofit/hs3/src/HS3Parser.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 543d9e9b71e7c..a4696afc42c41 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -37,8 +37,8 @@ namespace { // so it has no additive-linear or multiplicative-linear poly6 variant. struct InterpolationCodes { Interpolation descriptor; - int piecewise; - int flexible; + int piecewise = 0; + int flexible = 0; }; constexpr int kUnrepresentable = -1; From fcc9a380fba90ce46126d05e7c19747b26a015ec Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 13 Nov 2026 13:51:48 +0000 Subject: [PATCH 3/9] [RF][HS3] Enable `readability-inconsistent-declaration-parameter-name` Make the parameter names in the declarations match the ones in the definitions. In most cases this is done with the fix-its of the check, which rename the parameters in the declaration. Where the name in the header was the better one, the parameter in the definition was renamed by hand instead: * HS3: readValue() for ParameterValue, importHistFactoryNode() Assisted-by: claude-code:claude-fable-5-2 Cherry-picked from 82e73e36df1 --- roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 8 ++++---- roofit/hs3/src/HS3Parser.cxx | 16 ++++++++-------- roofit/hs3/src/JSONFactories_HistFactory.cxx | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 291be7fc4a7cf..f0fec52ad1a9c 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -100,7 +100,7 @@ class RooJSONFactoryWSTool { static std::string name(const RooFit::Detail::JSONNode &n); static bool isValidName(const std::string &str); - static bool testValidName(const std::string &str, bool forcError); + static bool testValidName(const std::string &name, bool forceError); static std::string sanitizeName(const std::string str); static void rebuildModelConfigInWorkspace(RooStats::ModelConfig *mc, RooWorkspace &ws); @@ -191,14 +191,14 @@ class RooJSONFactoryWSTool { readBinnedData(const RooFit::Detail::JSONNode &n, const std::string &namecomp, RooArgSet const &vars); bool importJSON(std::string const &filename); - bool importJSON(std::istream &os); + bool importJSON(std::istream &is); bool exportJSON(std::string const &fileName); bool exportJSON(std::ostream &os); std::string exportJSONtoString(); bool importJSONfromString(const std::string &s); void importJSONElement(const std::string &name, const std::string &jsonString); - void importVariableElement(const RooFit::Detail::JSONNode &n); + void importVariableElement(const RooFit::Detail::JSONNode &elementNode); void importFunction(const RooFit::Detail::JSONNode &p, bool importAllDependants); void importFunction(const std::string &jsonString, bool importAllDependants); @@ -302,7 +302,7 @@ class RooJSONFactoryWSTool { void exportAllObjects(RooFit::Detail::JSONNode &n); void exportModelConfig(RooStats::ModelConfig const &mc, - const std::vector &combined, + const std::vector &combDataSets, const std::vector &single); void exportSingleModelConfig(RooStats::ModelConfig const &mc, std::string const &analysisName, diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index a4696afc42c41..433507936bf9e 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -439,30 +439,30 @@ void describe(Reader &ar, Domain &domain) ar.list("axes", domain.axes); } -void readValue(Cursor const &c, ParameterValue &p) +void readValue(Cursor const &c, ParameterValue &out) { if (!c.node().is_map()) { c.error("must be a struct"); return; } Reader ar{c}; - ar("name", p.name); + ar("name", out.name); if (ar.has("value")) { Cursor crsr{ar.cursor().node()["value"], ar.cursor(), "value"}; if (crsr.node().is_number()) { - p.value = crsr.node().val_double(); + out.value = crsr.node().val_double(); } else { std::string label; readValue(crsr, label); - p.value = std::move(label); + out.value = std::move(label); } } std::optional isConst; ar("const", isConst); - p.isConst = isConst.value_or(false); - ar("err", p.error); - ar("relErr", p.relError); - ar("nbins", p.nbins); + out.isConst = isConst.value_or(false); + ar("err", out.error); + ar("relErr", out.relError); + ar("nbins", out.nbins); } void describe(Reader &ar, ParameterPoint &point) diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 5394884fcbc46..6281f2f64ffd5 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -1927,15 +1927,15 @@ bool RooFit::HS3::isBuiltinHistFactoryImporter(RooFit::JSONIO::Importer const &i } // The upgrade rewrites the JSON, so it has to work on a copy. -void RooFit::HS3::importHistFactoryNode(RooJSONFactoryWSTool &tool, JSONNode const &p, +void RooFit::HS3::importHistFactoryNode(RooJSONFactoryWSTool &tool, JSONNode const &node, std::vector const *channelConstraints) { - std::string name = RooJSONFactoryWSTool::name(p); - if (!p.has_child("samples")) { + std::string name = RooJSONFactoryWSTool::name(node); + if (!node.has_child("samples")) { RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); } std::stringstream ss; - p.writeJSON(ss); + node.writeJSON(ss); std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); HS3::Version version; From cb263a75342ae82130752a835554cafe4a104e4e Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 19 Nov 2026 11:12:23 +0100 Subject: [PATCH 4/9] [RF][HS3] Refactor and dead code cleanup to reduce lines of code (#24231) RooFitHS3 grew a lot when it was synced with the upcoming HS3 0.3 standard and moved to the typed document representation. Now that the right abstractions and the required public interface are clearer, this reduces roofit/hs3 by about 1000 lines without intended changes in functionality. The exported documents are the same, and the tests pass without changes. Mostly, this removes unused functions and code that repeated what the parser, the writer and the validation of the document already do. The check for shared variates moves to the new HS3Variates.cxx in hs3cpp, which is compiled without RooFit. Removed from the public headers; what was in a release is listed in the release notes: * RooJSONFactoryWSTool: createNewJSONTree(), makeVariablesNode(), getRooFitInternal(), exportAxis(), exportCategoricalAxis(), and rebuildModelConfigInWorkspace(), which was only declared. hs3VersionInput() is replaced by hs3Version(), which gives the parsed version instead of the string. * JSONNode: has_key(), operator>>(std::string &), the fill_seq() overloads with a maximum size or a transformation, the decrement and arrow operators of the child iterators, and the operators that stream a map into a node, the children of a non-const node into a std::vector, or a node into a std::ostream. What does change: * removeExporters() and removeImporters() find the built-in ones by the names of their classes, which are different now. The exporters of JSONFactories_RooFitCore.cxx match the exported class (e.g. "RooGaussian" instead of "exportGaussian"), its function importers share one class, and the HistFactory exporters are HistFactoryStreamer instead of HistFactoryStreamer_ProdPdf and HistFactoryStreamer_SumPdf. * Diagnostics: malformed "axes" of a generic formula get the messages of the parser, which warns about unknown components and reads null as left out. Interpolations in error messages have their components sorted by name, and some diagnostics come in a different order. * HistFactory export: if a sample has several ParamHistFuncs named "mc_stat_*", the constraints of all but the last one are not written as constraint terms of the channel anymore. HistFactory doesn't create such models. * Variates: in a HistFactory distribution that the document interprets, the "constraints" of a custom modifier don't count as terms of the likelihood anymore. A combined distribution that no likelihood lists is also checked if a likelihood lists a data set of the same name. Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit 0d79bf77b8461385d5a9234cfd407a5ff2520a10) --- roofit/hs3/CMakeLists.txt | 1 + roofit/hs3/inc/RooFit/Detail/JSONInterface.h | 64 - .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 22 +- roofit/hs3/src/HS3Builder.h | 9 +- roofit/hs3/src/HS3Document.cxx | 5 - roofit/hs3/src/HS3Document.h | 64 +- roofit/hs3/src/HS3Parser.cxx | 301 ++-- roofit/hs3/src/HS3Schema.h | 52 +- roofit/hs3/src/HS3Validate.cxx | 37 +- roofit/hs3/src/HS3Variates.cxx | 485 +++++++ roofit/hs3/src/HS3Writer.cxx | 65 +- roofit/hs3/src/JSONFactories_HistFactory.cxx | 500 ++----- roofit/hs3/src/JSONFactories_RooFitCore.cxx | 1112 ++++++--------- roofit/hs3/src/JSONIOUtils.cxx | 59 +- roofit/hs3/src/JSONIOUtils.h | 28 +- roofit/hs3/src/JSONInterface.cxx | 20 - roofit/hs3/src/RooJSONFactoryWSTool.cxx | 1251 ++++------------- 17 files changed, 1544 insertions(+), 2531 deletions(-) create mode 100644 roofit/hs3/src/HS3Variates.cxx diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index ab3af61ff88ac..b1e1b041f0a4b 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -16,6 +16,7 @@ add_library(hs3cpp OBJECT src/HS3Parser.cxx src/HS3Upgrade.cxx src/HS3Validate.cxx + src/HS3Variates.cxx src/HS3Writer.cxx src/JSONInterface.cxx ) diff --git a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h index b9dfb0a913be6..59e63ddb65b59 100644 --- a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h +++ b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h @@ -14,8 +14,6 @@ #define RooFit_Detail_JSONInterface_h #include -#include -#include #include #include #include @@ -37,13 +35,7 @@ class JSONNode { ++_pos; return *this; } - child_iterator_t &operator--() - { - --_pos; - return *this; - } Nd &operator*() const { return **_pos; } - Nd &operator->() const { return **_pos; } friend bool operator!=(child_iterator_t const &lhs, child_iterator_t const &rhs) { return lhs._pos != rhs._pos; } friend bool operator==(child_iterator_t const &lhs, child_iterator_t const &rhs) { return lhs._pos == rhs._pos; } @@ -79,7 +71,6 @@ class JSONNode { JSONNode &operator<<(int i); JSONNode &operator<<(double d); JSONNode &operator<<(bool b); - const JSONNode &operator>>(std::string &v) const; JSONNode &operator[](std::string const &k); const JSONNode &operator[](std::string const &k) const; bool is_container() const; @@ -103,7 +94,6 @@ class JSONNode { std::optional try_val_bool() const; template T val_t() const; - bool has_key() const; bool has_val() const; bool has_child(std::string const &) const; JSONNode &append_child(); @@ -126,28 +116,6 @@ class JSONNode { } } - template - void fill_seq(Collection const &coll, size_t nmax) - { - set_seq(); - size_t n = 0; - for (auto const &item : coll) { - if (n >= nmax) - break; - append_child() << item; - ++n; - } - } - - template - void fill_seq(Collection const &coll, TransformationFunc func) - { - set_seq(); - for (auto const &item : coll) { - append_child() << func(item); - } - } - template void fill_mat(Matrix const &mat) { @@ -219,17 +187,6 @@ class JSONTree { std::unique_ptr _storage; }; -std::ostream &operator<<(std::ostream &os, RooFit::Detail::JSONNode const &s); - -template -std::vector &operator<<(std::vector &v, RooFit::Detail::JSONNode::children_view const &cv) -{ - for (const auto &e : cv) { - v.push_back(e.val_t()); - } - return v; -} - template std::vector &operator<<(std::vector &v, RooFit::Detail::JSONNode::const_children_view const &cv) { @@ -249,27 +206,6 @@ std::vector &operator<<(std::vector &v, RooFit::Detail::JSONNode const &n) return v; } -template -RooFit::Detail::JSONNode & -operator<<(RooFit::Detail::JSONNode &n, const std::unordered_map &m) -{ - n.set_map(); - for (const auto &it : m) { - n[it.first] << it.second; - } - return n; -} - -template -RooFit::Detail::JSONNode &operator<<(RooFit::Detail::JSONNode &n, const std::map &m) -{ - n.set_map(); - for (const auto &it : m) { - n[it.first] << it.second; - } - return n; -} - template <> inline int JSONNode::val_t() const { diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index f0fec52ad1a9c..4af617545fbe3 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -35,6 +35,7 @@ struct ParameterPoint; struct ParameterValue; struct ToolExportState; struct ToolImportState; +struct Version; template struct Ref; using DistributionRef = Ref; @@ -47,7 +48,6 @@ class ModelConfig; } class RooRealVar; class RooAbsCategory; -class RooCategory; class RooSimultaneous; class RooJSONFactoryWSTool { @@ -102,7 +102,6 @@ class RooJSONFactoryWSTool { static bool isValidName(const std::string &str); static bool testValidName(const std::string &name, bool forceError); static std::string sanitizeName(const std::string str); - static void rebuildModelConfigInWorkspace(RooStats::ModelConfig *mc, RooWorkspace &ws); static RooFit::Detail::JSONNode &appendNamedChild(RooFit::Detail::JSONNode &node, std::string const &name); static RooFit::Detail::JSONNode const *findNamedChild(RooFit::Detail::JSONNode const &node, std::string const &name); @@ -203,10 +202,6 @@ class RooJSONFactoryWSTool { void importFunction(const RooFit::Detail::JSONNode &p, bool importAllDependants); void importFunction(const std::string &jsonString, bool importAllDependants); - static std::unique_ptr createNewJSONTree(); - - static RooFit::Detail::JSONNode &makeVariablesNode(RooFit::Detail::JSONNode &rootNode); - // error handling helpers class DependencyMissingError : public std::runtime_error { std::string _parent, _child, _class; @@ -224,15 +219,6 @@ class RooJSONFactoryWSTool { const std::string &classname() const { return _class; } }; - template - static RooFit::Detail::JSONNode &getRooFitInternal(RooFit::Detail::JSONNode &node, Keys_t const &...keys) - { - return node.get("misc", "ROOT_internal", keys...); - } - - static void exportAxis(RooFit::Detail::JSONNode &obsNode, RooRealVar const &var); - static void exportCategoricalAxis(RooFit::Detail::JSONNode &obsNode, RooCategory const &cat); - static void exportHisto(RooArgSet const &vars, std::size_t n, double const *contents, RooFit::Detail::JSONNode &output); @@ -260,8 +246,9 @@ class RooJSONFactoryWSTool { std::string getStringAttribute(const std::string &obj, const std::string &attrib); void setStringAttribute(const std::string &obj, const std::string &attrib, const std::string &value); - /// The "hs3_version" of the document that is currently being imported, or an empty string outside of an import. - std::string const &hs3VersionInput() const { return _hs3VersionInput; } + /// The HS3 version of the document that is currently being imported. Outside of an import it is 0.0.0, like a + /// version that can't be parsed. + RooFit::HS3::Version hs3Version() const; private: template @@ -312,7 +299,6 @@ class RooJSONFactoryWSTool { std::vector channelConstraints); // member variables - std::string _hs3VersionInput; RooWorkspace &_workspace; // objects to represent intermediate information diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h index dda8410ddc2a9..718f5cc46c6e5 100644 --- a/roofit/hs3/src/HS3Builder.h +++ b/roofit/hs3/src/HS3Builder.h @@ -22,6 +22,7 @@ #include class RooAbsArg; +class RooAbsBinning; class RooDataHist; class RooJSONFactoryWSTool; class RooRealVar; @@ -33,14 +34,16 @@ class Importer; namespace HS3 { -/// Forward the warnings to the message service, and throw if there are errors. -void report(Diagnostics const &diagnostics); +/// Forward the warnings to the message service, and throw if there are errors. The paths of the diagnostics start with +/// the name of the object, if they are about an object that doesn't come with a document. +void report(Diagnostics const &diagnostics, std::string const &object = ""); /// Create the variables for the axes of binned data. RooArgSet createObservables(std::vector const &axes); -/// The axis of binned data for a variable. +/// The axis of binned data for a variable, or for a binning of a variable with the given name. BinnedAxis toBinnedAxis(RooRealVar const &var); +BinnedAxis toBinnedAxis(std::string const &name, RooAbsBinning const &binning); /// The name of an object, or the value of a literal constant. NameOrNumber toNameOrNumber(RooAbsArg const &arg); diff --git a/roofit/hs3/src/HS3Document.cxx b/roofit/hs3/src/HS3Document.cxx index bb3498a84a6cd..c89bf192cb89f 100644 --- a/roofit/hs3/src/HS3Document.cxx +++ b/roofit/hs3/src/HS3Document.cxx @@ -53,11 +53,6 @@ DomainAxis const *DomainRanges::find(std::string const &name) const namespace { -std::string const &nameOf(Modifier const &modifier) -{ - return std::visit([](auto const &m) -> std::string const & { return m.name; }, modifier); -} - char const *typeOf(Modifier const &modifier) { return std::visit( diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index c4a9c120a23fb..b2c27d4353605 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -41,13 +42,7 @@ struct Version { inline bool operator<(Version const &l, Version const &r) { - if (l.major != r.major) { - return l.major < r.major; - } - if (l.minor != r.minor) { - return l.minor < r.minor; - } - return l.patch < r.patch; + return std::tie(l.major, l.minor, l.patch) < std::tie(r.major, r.minor, r.patch); } /// Combines lambdas into one visitor for std::visit(). The compiler complains about the alternatives without lambda. @@ -58,6 +53,13 @@ struct overloaded : Ts... { template overloaded(Ts...) -> overloaded; +/// The name of the alternative that a variant holds, e.g. of an axis or of a modifier. +template +std::string const &nameOf(std::variant const &v) +{ + return std::visit([](auto const &alternative) -> std::string const & { return alternative.name; }, v); +} + /// A name that says what it refers to, so that for example a DataRef can't be used as a DomainRef. template struct Ref { @@ -229,6 +231,8 @@ struct Analysis { /// its type in RooFit::JSONIO::importers(). struct GenericObject { std::string type; + /// The object in the JSON tree that the document was read from. In a document that is exported, the distributions + /// refer to what the exporters wrote once all of them are written, and the functions don't have a node. Detail::JSONNode const *node = nullptr; }; @@ -261,13 +265,7 @@ inline bool operator!=(Interpolation const &l, Interpolation const &r) } inline bool operator<(Interpolation const &l, Interpolation const &r) { - if (l.type != r.type) { - return l.type < r.type; - } - if (l.in != r.in) { - return l.in < r.in; - } - return l.out < r.out; + return std::tie(l.type, l.in, l.out) < std::tie(r.type, r.in, r.out); } inline constexpr Interpolation additivePiecewiseLinear{InterpolationType::Add, InterpolationFunction::Poly1, @@ -521,7 +519,7 @@ std::optional validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options); } // namespace HS3 -/// An HS3 document. It is move-only, because generic objects refer to the JSON tree that the document was read from. +/// An HS3 document. It is move-only, because generic objects refer to the JSON tree of the document. class HS3Document { public: HS3::Metadata metadata; @@ -566,6 +564,9 @@ HS3Document parseDocument(Detail::JSONNode const &root, Diagnostics &diagnostics /// Reads a HistFactory distribution that doesn't come with a document. HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagnostics &diagnostics); +/// Reads a data set that doesn't come with a document. +Data parseData(Detail::JSONNode const &data, Diagnostics &diagnostics); + /// Reads the axes of binned data that doesn't come with a document. std::vector parseBinnedAxes(Detail::JSONNode const &axes, Diagnostics &diagnostics); @@ -581,6 +582,12 @@ void writeDocument(Detail::JSONNode &root, HS3Document const &doc); /// Writes a HistFactory distribution, without its name. void writeHistFactoryDist(Detail::JSONNode &node, HistFactoryDist const &dist); +/// Writes the axes, contents and uncertainties of binned data, without its name and type. +void writeBinnedData(Detail::JSONNode &node, BinnedData const &data); + +/// Writes an axis of binned data. +void writeAxis(Detail::JSONNode &node, BinnedAxis const &axis); + /// Writes the functions "interpolation" and "interpolation0d", with a single interpolation if they are all the same. void writeInterpolatedFunction(Detail::JSONNode &node, InterpolatedFunction const &function); void writeInterpolatedNumber(Detail::JSONNode &node, InterpolatedNumber const &function); @@ -641,6 +648,33 @@ void mergeDuplicateModifiers(HistFactoryDist &dist, std::string const &channel, /// Makes the most common interpolation of the modifiers the "default_interpolation", which they then don't repeat. void factorOutDefaultInterpolation(HistFactoryDist &dist); +// HS3 requires the terms of a likelihood, and the components of a combined distribution or data set, to have distinct +// variates. For this, the generic distributions are read as far as the standard says what they consist of (e.g. the +// factors of a product), through the nodes that the document refers to. + +/// Describes each variate that is shared although it must be distinct: by the terms of a likelihood, by the +/// components of a combined distribution that no likelihood lists (the others are checked through the likelihood), or +/// by the components of a combined data set, except for bookkeeping columns. +std::vector sharedVariates(HS3Document const &doc); + +/// The variates of the "distributions" of a likelihood, which it pairs with data, with the columns of its combined +/// distributions. +std::set variatesOf(HS3Document const &doc, Likelihood const &likelihood); + +/// The axes that several components of a combined data set have, by combined data set. +std::map> sharedAxes(HS3Document const &doc); + +/// The bookkeeping columns by combined data set: the declared ones (RooFitInternal::bookkeepingColumns), except for +/// those that are a variate of a distribution after all. They are exempt from the requirement of distinct variates. +std::map> bookkeepingColumns(HS3Document const &doc); + +/// Writes each term of the likelihoods once: a factor of a product that is a term of the likelihood as well would enter +/// it twice, so it is removed from the "factors" of the product in `root`, the JSON tree that the document is written +/// to. Only products that no other distribution uses are changed (being a component of a combined distribution is not +/// a use), only factors that are terms of every likelihood that has the product as a term, and a product whose factors +/// are all such terms is left as it is. Returns the removed factors by product. +std::map> writeLikelihoodTermsOnce(HS3Document const &doc, Detail::JSONNode &root); + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 433507936bf9e..7452ed3b35206 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -11,7 +11,6 @@ */ #include "HS3Schema.h" -#include "JSONIOUtils.h" #include #include @@ -150,7 +149,8 @@ Reader::~Reader() { JSONNode const &node = _cursor.node(); // There is nothing to report if all components were read, and after an error it would only be noise. - if (!node.is_map() || node.num_children() == _seen.size() || _cursor.numErrors() != _nErrorsBefore) { + if (!_reportUnread || !node.is_map() || node.num_children() == _seen.size() || + _cursor.numErrors() != _nErrorsBefore) { return; } for (JSONNode const &child : node.children()) { @@ -240,7 +240,7 @@ void readBound(Reader &ar, const char *key, std::optional &out, double v if (!ar.has(key)) { return; } - Cursor c{ar.cursor().node()[key], ar.cursor(), key}; + Cursor c = ar.child(key); if (c.node().is_null()) { out = valueIfNull; return; @@ -254,7 +254,7 @@ constexpr double infinity = std::numeric_limits::infinity(); // Read bin edges, which must be at least two strictly ascending numbers. void readEdges(Reader &ar, std::string const &axisName, std::vector &edges) { - Cursor c{ar.cursor().node()["edges"], ar.cursor(), "edges"}; + Cursor c = ar.child("edges"); const std::string context = "\"edges\" of axis '" + axisName + "' must "; if (!c.node().is_seq()) { c.error(context + "be a list"); @@ -283,16 +283,17 @@ void describe(Reader &ar, Package &p) void describe(Reader &ar, DomainAxis &axis) { ar("name", axis.name); - if (ar.has(fixedValueKey)) { - // The domain of the parameter is the set with this single value: no range, binning or categories. + // A fixed axis (HS3 #131): the domain of the parameter is the set with this single value, without range, binning or + // categories. + if (ar.has("value")) { for (const char *key : {"min", "max", "nbins", "edges", "categories"}) { if (ar.has(key)) { - ar.cursor().error("the axis '" + axis.name + "' has \"" + fixedValueKey + "\" and \"" + key + + ar.cursor().error("the axis '" + axis.name + "' has \"value\" and \"" + key + "\", which exclude each other"); } } // Not read as an optional component, for which null would be the same as leaving it out. - ar(fixedValueKey, axis.fixedValue.emplace()); + ar("value", axis.fixedValue.emplace()); return; } readBound(ar, "min", axis.min, -infinity); @@ -385,70 +386,35 @@ void describe(Reader &ar, CategoricalAxis &axis) } } -namespace { - // The kind of an axis is determined by its components. -template -void readAxis(Cursor const &c, std::variant &out) +template +void describe(Reader &ar, std::variant &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader reader{c}; // A fixed axis is not categorical. That it has no categories is checked when it is read. - if (reader.has("categories") && !(std::is_same_v && reader.has(fixedValueKey))) { - CategoricalAxis axis; - describe(reader, axis); - out = std::move(axis); + if (ar.has("categories") && !(std::is_same_v && ar.has("value"))) { + describe(ar, out.template emplace()); } else { - ContinuousAxis_t axis; - describe(reader, axis); - out = std::move(axis); + describe(ar, out.template emplace()); } } -} // namespace - -void readValue(Cursor const &c, std::variant &out) -{ - readAxis(c, out); -} - -void readValue(Cursor const &c, std::variant &out) -{ - readAxis(c, out); -} - -void readValue(Cursor const &c, std::variant &out) -{ - readAxis(c, out); -} - void describe(Reader &ar, Domain &domain) { ar("name", domain.name); std::string type; - if (ar.has("type")) { - ar("type", type); - } + ar.optional("type", type); if (type != "product_domain") { ar.cursor().error("only domains of type \"product_domain\" are currently supported!"); return; } - ar.list("axes", domain.axes); + ar.optional("axes", domain.axes); } -void readValue(Cursor const &c, ParameterValue &out) +void describe(Reader &ar, ParameterValue &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; ar("name", out.name); if (ar.has("value")) { - Cursor crsr{ar.cursor().node()["value"], ar.cursor(), "value"}; + Cursor crsr = ar.child("value"); if (crsr.node().is_number()) { out.value = crsr.node().val_double(); } else { @@ -457,9 +423,7 @@ void readValue(Cursor const &c, ParameterValue &out) out.value = std::move(label); } } - std::optional isConst; - ar("const", isConst); - out.isConst = isConst.value_or(false); + ar.optional("const", out.isConst); ar("err", out.error); ar("relErr", out.relError); ar("nbins", out.nbins); @@ -468,19 +432,19 @@ void readValue(Cursor const &c, ParameterValue &out) void describe(Reader &ar, ParameterPoint &point) { ar("name", point.name); - ar.list("parameters", point.parameters); + ar.optional("parameters", point.parameters); } void describe(Reader &ar, PointData &data) { ar("value", data.value); ar("uncertainty", data.uncertainty); - ar.list("axes", data.axes); + ar.optional("axes", data.axes); } void describe(Reader &ar, UnbinnedData &data) { - ar.list("axes", data.axes); + ar.optional("axes", data.axes); ar("weights", data.weights); data.hasEntriesUncertainties = ar.has("entries_uncertainties"); @@ -488,7 +452,7 @@ void describe(Reader &ar, UnbinnedData &data) ar.cursor().error("does not define the required component \"entries\""); return; } - Cursor entries{ar.cursor().node()["entries"], ar.cursor(), "entries"}; + Cursor entries = ar.child("entries"); if (!entries.node().is_seq()) { entries.error("key 'entries' is not a list!"); return; @@ -543,7 +507,7 @@ void describe(Reader &ar, GaussianUncertainty &uncertainty) } ar("sigma", uncertainty.sigma); if (ar.has("correlation")) { - Cursor c{ar.cursor().node()["correlation"], ar.cursor(), "correlation"}; + Cursor c = ar.child("correlation"); // Besides a matrix, it can be the number 0 to indicate that there is no correlation. if (c.node().is_seq()) { readValue(c, uncertainty.correlation); @@ -555,12 +519,12 @@ void describe(Reader &ar, GaussianUncertainty &uncertainty) void describe(Reader &ar, BinnedData &data) { - ar.list("axes", data.axes); + ar.optional("axes", data.axes); ar("contents", data.contents); ar("uncertainty", data.uncertainty); // RooFit extension: older versions read the errors like in the samples of a HistFactory distribution. if (ar.has("errors")) { - Cursor errors{ar.cursor().node()["errors"], ar.cursor(), "errors"}; + Cursor errors = ar.child("errors"); if (!errors.node().is_seq()) { errors.error("errors are not in list form"); } else if (!data.uncertainty) { @@ -624,41 +588,28 @@ void readCombinedData(Reader &ar, std::string const &name, CombinedData &data) } } -void readValue(Cursor const &c, Data &out) +void describe(Reader &ar, Data &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; ar("name", out.name); std::string type; ar("type", type); if (type == "binned") { - BinnedData body; - describe(ar, body); - out.body = std::move(body); + describe(ar, out.body.emplace()); } else if (type == "unbinned") { - UnbinnedData body; - describe(ar, body); - out.body = std::move(body); + describe(ar, out.body.emplace()); } else if (type == "point") { - PointData body; - describe(ar, body); - out.body = std::move(body); + describe(ar, out.body.emplace()); } else if (type == "combined") { - CombinedData body; - readCombinedData(ar, out.name, body); - out.body = std::move(body); + readCombinedData(ar, out.name, out.body.emplace()); } else if (ar.has("type")) { - c.error("unknown type of dataset '" + type + "'"); + ar.cursor().error("unknown type of dataset '" + type + "'"); } } void describe(Reader &ar, Likelihood &likelihood) { ar("name", likelihood.name); - ar.list("aux_distributions", likelihood.auxDistributions); + ar.optional("aux_distributions", likelihood.auxDistributions); std::vector distributions; ar("distributions", distributions); @@ -666,7 +617,7 @@ void describe(Reader &ar, Likelihood &likelihood) ar.cursor().error("does not define the required component \"data\""); return; } - Cursor data{ar.cursor().node()["data"], ar.cursor(), "data"}; + Cursor data = ar.child("data"); if (!data.node().is_seq()) { data.error("must be an array"); return; @@ -707,13 +658,13 @@ void describe(Reader &ar, Analysis &analysis) "' is deprecated, please use \"" + key + "\" instead."); return true; }; - ar.list(isLegacy("parameters_of_interest", "parameter_of_interest") ? "parameter_of_interest" - : "parameters_of_interest", - analysis.parametersOfInterest); + ar.optional(isLegacy("parameters_of_interest", "parameter_of_interest") ? "parameter_of_interest" + : "parameters_of_interest", + analysis.parametersOfInterest); if (!isLegacy("domains", "domain")) { - ar.list("domains", analysis.domains); + ar.optional("domains", analysis.domains); } else if (ar.cursor().node()["domain"].is_seq()) { - ar.list("domain", analysis.domains); + ar.optional("domain", analysis.domains); } else { analysis.domains.emplace_back(); ar("domain", analysis.domains.back()); @@ -844,29 +795,33 @@ std::optional readConstraintType(Cursor const &c) return std::nullopt; } +// The parameter of a modifier with a single parameter and its constraint are arrays with one entry. +template +void readSingleEntry(Reader &ar, const char *key, T &out) +{ + Cursor c = ar.child(key); + if (!c.node().is_seq() || c.node().num_children() != 1) { + c.error("must be an array with one entry"); + return; + } + readValue(Cursor{c.node().child(0), c, std::size_t{0}}, out); +} + void readConstraint(Reader &ar, Constraint &out) { if (ar.has("constraints")) { - Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; - if (!c.node().is_seq() || c.node().num_children() != 1) { - c.error("must be an array with one entry"); - return; + std::optional pdf; + readSingleEntry(ar, "constraints", pdf); + if (pdf) { + out = ConstraintPdf{std::move(*pdf)}; } - JSONNode const &item = c.node().child(0); - if (item.is_null()) { - out = Unconstrained{}; - return; - } - ConstraintPdf pdf; - readValue(Cursor{item, c, std::size_t{0}}, pdf.pdf); - out = std::move(pdf); } else if (ar.has("constraint_name")) { ConstraintPdf pdf; ar("constraint_name", pdf.pdf); pdf.legacyName = true; out = std::move(pdf); } else if (ar.has("constraint")) { - if (auto type = readConstraintType(Cursor{ar.cursor().node()["constraint"], ar.cursor(), "constraint"})) { + if (auto type = readConstraintType(ar.child("constraint"))) { out = *type; } } @@ -875,24 +830,9 @@ void readConstraint(Reader &ar, Constraint &out) void readGammaConstraint(Reader &ar, GammaConstraint &out) { if (ar.has("constraints")) { - GammaConstraintPdfs constraints; - Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; - if (!c.node().is_seq()) { - c.error("must be an array"); - return; - } - std::size_t i = 0; - for (JSONNode const &item : c.node().children()) { - constraints.pdfs.emplace_back(); - if (!item.is_null()) { - constraints.pdfs.back().emplace(); - readValue(Cursor{item, c, i}, *constraints.pdfs.back()); - } - ++i; - } - out = std::move(constraints); + ar("constraints", out.emplace().pdfs); } else if (ar.has("constraint")) { - Cursor c{ar.cursor().node()["constraint"], ar.cursor(), "constraint"}; + Cursor c = ar.child("constraint"); // "Const" is a RooFit extension. if (!c.node().is_container() && !c.node().is_null() && c.node().val() == "Const") { out = ConstantGammas{}; @@ -904,39 +844,27 @@ void readGammaConstraint(Reader &ar, GammaConstraint &out) } // namespace -// The standard doesn't require a name for modifiers, but RooFit uses it to name the objects it creates. -void readModifierName(Reader &ar, std::string &name) -{ - std::optional optionalName; - ar("name", optionalName); - name = optionalName.value_or(""); -} - // The parameter of a modifier with a single parameter is given in "parameters", or in "parameter" in older documents. void readParameter(Reader &ar, ParameterRef ¶meter) { - if (!ar.has("parameters")) { + if (ar.has("parameters")) { + readSingleEntry(ar, "parameters", parameter); + } else { ar("parameter", parameter); - return; } - Cursor c{ar.cursor().node()["parameters"], ar.cursor(), "parameters"}; - if (!c.node().is_seq() || c.node().num_children() != 1) { - c.error("must be an array with one entry"); - return; - } - readValue(Cursor{c.node().child(0), c, std::size_t{0}}, parameter); } +// The modifiers. The standard doesn't require them to have a name, but RooFit uses it to name the objects it creates. void describe(Reader &ar, NormFactor &m) { - readModifierName(ar, m.name); + ar.optional("name", m.name); readParameter(ar, m.parameter); readConstraint(ar, m.constraint); } void describe(Reader &ar, NormSys &m) { - readModifierName(ar, m.name); + ar.optional("name", m.name); readParameter(ar, m.parameter); ar.nested("data", [&](Reader &data) { data("hi", m.hi); @@ -954,7 +882,7 @@ void describe(Reader &ar, HistoSysVariation &variation) void describe(Reader &ar, HistoSys &m) { - readModifierName(ar, m.name); + ar.optional("name", m.name); readParameter(ar, m.parameter); ar.nested("data", [&](Reader &data) { data("hi", m.hi); @@ -966,22 +894,22 @@ void describe(Reader &ar, HistoSys &m) void describe(Reader &ar, ShapeSys &m) { - readModifierName(ar, m.name); + ar.optional("name", m.name); if (ar.has("parameter")) { // A single name stands for one parameter per bin, which are named after it. ar("parameter", m.name); } - ar.list("parameters", m.parameters); + ar.optional("parameters", m.parameters); if (ar.has("data")) { - ar.nested("data", [&](Reader &data) { data.list("vals", m.vals); }); + ar.nested("data", [&](Reader &data) { data.optional("vals", m.vals); }); } readGammaConstraint(ar, m.constraint); } void describe(Reader &ar, StatError &m) { - readModifierName(ar, m.name); - ar.list("parameters", m.parameters); + ar.optional("name", m.name); + ar.optional("parameters", m.parameters); readGammaConstraint(ar, m.constraint); } @@ -990,37 +918,28 @@ void describe(Reader &ar, CustomModifier &m) ar("name", m.name); } -void readValue(Cursor const &c, Modifier &out) +void describe(Reader &ar, Modifier &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; std::string type; ar("type", type); - auto read = [&](auto modifier) { - describe(ar, modifier); - out = std::move(modifier); - }; if (type == "normfactor") { - read(NormFactor{}); + describe(ar, out.emplace()); } else if (type == "normsys") { - read(NormSys{}); + describe(ar, out.emplace()); } else if (type == "histosys") { - read(HistoSys{}); + describe(ar, out.emplace()); } else if (type == "shapesys" || type == "shapefactor") { - read(ShapeSys{}); + describe(ar, out.emplace()); } else if (type == "staterror") { - read(StatError{}); + describe(ar, out.emplace()); } else if (type == "custom") { - read(CustomModifier{}); + describe(ar, out.emplace()); } else if (ar.has("type")) { std::string name; if (ar.has("name")) { ar("name", name); } - c.error("modifier '" + name + "' of unknown type '" + type + "'"); + ar.cursor().error("modifier '" + name + "' of unknown type '" + type + "'"); } } @@ -1031,19 +950,15 @@ void describe(Reader &ar, Sample &sample) data("contents", sample.contents); data("errors", sample.errors); }); - ar.list("modifiers", sample.modifiers); + ar.optional("modifiers", sample.modifiers); } void describe(Reader &ar, StatErrorConfig &config) { - std::optional relThreshold; - ar("relThreshold", relThreshold); - config.relThreshold = relThreshold.value_or(0.0); + ar.optional("relThreshold", config.relThreshold); readGammaConstraint(ar, config.constraint); // Written by upgradeDocument() for older documents. - std::optional impliedPoissonErrors; - ar("implied_poisson_errors", impliedPoissonErrors); - config.impliedPoissonErrors = impliedPoissonErrors.value_or(false); + ar.optional("implied_poisson_errors", config.impliedPoissonErrors); } void describe(Reader &ar, HistFactoryDist &dist) @@ -1061,7 +976,7 @@ void describe(Reader &ar, HistFactoryDist &dist) ar("default_interpolation", dist.defaultInterpolation); ar("samples", dist.samples); if (ar.has("staterror")) { - readValue(Cursor{ar.cursor().node()["staterror"], ar.cursor(), "staterror"}, dist.statError); + readValue(ar.child("staterror"), dist.statError); } } @@ -1084,34 +999,25 @@ void readDistribution(Cursor const &c, Distribution &out, ParseOptions const &op } else { out.body = GenericObject{type, &c.node()}; // The importers read the other components. - for (JSONNode const &child : c.node().children()) { - ar.has(child.key().c_str()); - } + ar.ignoreUnread(); } } } // namespace -void readValue(Cursor const &c, Function &out) +void describe(Reader &ar, Function &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; ar("name", out.name); ar("type", out.body.type); - out.body.node = &c.node(); + out.body.node = &ar.cursor().node(); // The importers read the other components. - for (JSONNode const &child : c.node().children()) { - ar.has(child.key().c_str()); - } + ar.ignoreUnread(); } void describe(Reader &ar, Attributes &attributes) { if (ar.has("dict")) { - Cursor dict{ar.cursor().node()["dict"], ar.cursor(), "dict"}; + Cursor dict = ar.child("dict"); if (!dict.node().is_map()) { dict.error("must be a struct"); return; @@ -1120,7 +1026,7 @@ void describe(Reader &ar, Attributes &attributes) readValue(Cursor{item, dict, item.key().c_str()}, attributes.dict[item.key()]); } } - ar.list("tags", attributes.tags); + ar.optional("tags", attributes.tags); if (ar.has("is_const_var")) { int isConstVar = 0; ar("is_const_var", isConstVar); @@ -1165,7 +1071,7 @@ void readMap(Reader &ar, const char *key, Map_t &out) if (!ar.has(key)) { return; } - Cursor c{ar.cursor().node()[key], ar.cursor(), key}; + Cursor c = ar.child(key); if (c.node().is_null()) { return; } @@ -1266,9 +1172,9 @@ void readMetadata(Cursor const &root, Metadata &out, ParseOptions const &options Reader ar{metadataCursor}; ar.has("hs3_version"); - ar.list("packages", out.packages); - ar.list("authors", out.authors); - ar.list("publications", out.publications); + ar.optional("packages", out.packages); + ar.optional("authors", out.authors); + ar.optional("publications", out.publications); ar("description", out.description); } @@ -1300,12 +1206,12 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO } } } - ar.list("functions", doc.functions); - ar.list("data", doc.data); - ar.list("likelihoods", doc.likelihoods); - ar.list("domains", doc.domains); - ar.list("parameter_points", doc.parameterPoints); - ar.list("analyses", doc.analyses); + ar.optional("functions", doc.functions); + ar.optional("data", doc.data); + ar.optional("likelihoods", doc.likelihoods); + ar.optional("domains", doc.domains); + ar.optional("parameter_points", doc.parameterPoints); + ar.optional("analyses", doc.analyses); readMisc(cursor, doc.rooFit); return doc; } @@ -1366,7 +1272,7 @@ void readInterpolations(Reader &ar, std::size_t nParameters, InterpolationClass { out.assign(nParameters, additivePiecewiseLinear); if (ar.has("interpolations")) { - Cursor c{ar.cursor().node()["interpolations"], ar.cursor(), "interpolations"}; + Cursor c = ar.child("interpolations"); const std::size_t errorsBefore = c.numErrors(); std::vector interpolations; readValue(c, interpolations); @@ -1387,7 +1293,7 @@ void readInterpolations(Reader &ar, std::size_t nParameters, InterpolationClass if (!ar.has("interpolationCodes")) { return; } - Cursor c{ar.cursor().node()["interpolationCodes"], ar.cursor(), "interpolationCodes"}; + Cursor c = ar.child("interpolationCodes"); if (!c.node().is_seq() || c.node().num_children() != nParameters) { c.error("legacy component 'interpolationCodes' must contain one code per parameter (got " + std::to_string(c.node().is_seq() ? c.node().num_children() : 0) + " for " + std::to_string(nParameters) + @@ -1433,6 +1339,13 @@ Function_t parseInterpolated(JSONNode const &node, Diagnostics &diagnostics, Int } // namespace +Data parseData(JSONNode const &data, Diagnostics &diagnostics) +{ + Data out; + readValue(Cursor{data, diagnostics}, out); + return out; +} + std::vector parseBinnedAxes(JSONNode const &axes, Diagnostics &diagnostics) { std::vector out; diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h index 6dfcebed75f60..a6020773a2378 100644 --- a/roofit/hs3/src/HS3Schema.h +++ b/roofit/hs3/src/HS3Schema.h @@ -70,9 +70,9 @@ class Reader { template void operator()(const char *key, std::optional &field); - /// Read an array that can be omitted, which is equivalent to an empty one. + /// Read a component that can be omitted or null, which leaves the field as it is, e.g. empty for an array. template - void list(const char *key, std::vector &field); + void optional(const char *key, T &field); /// Read the components of a struct that is nested in a component. template @@ -81,20 +81,24 @@ class Reader { /// Whether the struct has a component. Checking for a component counts as reading it. bool has(const char *key); + /// Don't report the components that were not read, because they are read later by someone else. + void ignoreUnread() { _reportUnread = false; } + Cursor const &cursor() const { return _cursor; } + /// The position of a component. + Cursor child(const char *key) const { return {_cursor.node()[key], _cursor, key}; } private: Cursor const &_cursor; std::vector _seen; std::size_t _nErrorsBefore = 0; + bool _reportUnread = true; }; void readValue(Cursor const &c, std::string &out); void readValue(Cursor const &c, double &out); void readValue(Cursor const &c, int &out); void readValue(Cursor const &c, bool &out); -void readValue(Cursor const &c, ParameterValue &out); -void readValue(Cursor const &c, Data &out); void readValue(Cursor const &c, Interpolation &out); void readValue(Cursor const &c, NameOrNumber &out); @@ -107,6 +111,15 @@ void readValue(Cursor const &c, Ref &out) readValue(c, out.name); } +/// A value that can be null, like the constraint of a parameter that is not constrained. +template +void readValue(Cursor const &c, std::optional &out) +{ + if (!c.node().is_null()) { + readValue(c, out.emplace()); + } +} + template void readValue(Cursor const &c, std::vector &out) { @@ -144,34 +157,23 @@ void Reader::operator()(const char *key, T &field) _cursor.error(std::string("does not define the required component \"") + key + "\""); return; } - readValue(Cursor{_cursor.node()[key], _cursor, key}, field); + readValue(child(key), field); } template void Reader::operator()(const char *key, std::optional &field) { - if (!has(key)) { - return; + if (has(key)) { + readValue(child(key), field); } - Detail::JSONNode const &node = _cursor.node()[key]; - if (node.is_null()) { - return; - } - field.emplace(); - readValue(Cursor{node, _cursor, key}, *field); } template -void Reader::list(const char *key, std::vector &field) +void Reader::optional(const char *key, T &field) { - if (!has(key)) { - return; - } - Detail::JSONNode const &node = _cursor.node()[key]; - if (node.is_null()) { - return; + if (has(key) && !_cursor.node()[key].is_null()) { + readValue(child(key), field); } - readValue(Cursor{node, _cursor, key}, field); } template @@ -181,12 +183,12 @@ void Reader::nested(const char *key, Fn &&fn) _cursor.error(std::string("does not define the required component \"") + key + "\""); return; } - Cursor child{_cursor.node()[key], _cursor, key}; - if (!child.node().is_map()) { - child.error("must be a struct"); + Cursor c = child(key); + if (!c.node().is_map()) { + c.error("must be a struct"); return; } - Reader reader{child}; + Reader reader{c}; fn(reader); } diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index b0696fe32ced9..133defab4c520 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -184,7 +184,7 @@ void Validator::collectNames() // HistFactory modifiers, which are created on the fly. for (auto const &domain : _doc.domains) { for (auto const &axis : domain.axes) { - addParameter(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); + addParameter(nameOf(axis)); } } for (auto const &point : _doc.parameterPoints) { @@ -241,8 +241,7 @@ void Validator::checkFixedAxes() for (std::size_t i = 0; i < _doc.domains.size(); ++i) { for (auto const &axis : _doc.domains[i].axes) { // A categorical axis is passed as null. - fn(i, std::visit([](auto const &a) -> std::string const & { return a.name; }, axis), - std::get_if(&axis)); + fn(i, nameOf(axis), std::get_if(&axis)); } } }; @@ -643,7 +642,9 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const ", contents=" + std::to_string(sample.contents.size())); } for (std::size_t iModifier = 0; iModifier < sample.modifiers.size(); ++iModifier) { - std::string modifierPath = samplePath + ".modifiers[" + std::to_string(iModifier) + "]"; + Modifier const &modifier = sample.modifiers[iModifier]; + const std::string modifierPath = + samplePath + ".modifiers[" + std::to_string(iModifier) + "](" + nameOf(modifier) + ")"; auto checkInterpolation = [&](std::optional const &interpolation, char const *type, std::string const &modifierName) { if (!interpolation && !dist.defaultInterpolation) { @@ -652,32 +653,26 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const "' is not given, and the channel has no \"default_interpolation\""); } }; + // The modifiers with a single parameter, which can have a constraint. + auto checkSingleParameter = [&](auto const &m) { + checkParameter(modifierPath, m.parameter.name, m.name); + checkConstraint(modifierPath, m.constraint); + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + }; std::visit(overloaded{ - [&](NormFactor const &m) { - modifierPath += "(" + m.name + ")"; - checkParameter(modifierPath, m.parameter.name, m.name); - checkConstraint(modifierPath, m.constraint); - checkCorrelated(modifierPath, m.parameter.name, m.constraint); - }, + [&](NormFactor const &m) { checkSingleParameter(m); }, [&](NormSys const &m) { - modifierPath += "(" + m.name + ")"; - checkParameter(modifierPath, m.parameter.name, m.name); - checkConstraint(modifierPath, m.constraint); - checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkSingleParameter(m); checkInterpolation(m.interpolation, "normsys", m.name); }, [&](HistoSys const &m) { - modifierPath += "(" + m.name + ")"; checkUniqueName(modifierPath, "histosys", m.name); - checkParameter(modifierPath, m.parameter.name, m.name); - checkConstraint(modifierPath, m.constraint); - checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkSingleParameter(m); checkInterpolation(m.interpolation, "histosys", m.name); checkSize(modifierPath + ".data.hi.contents", m.hi.contents.size(), "\"contents\""); checkSize(modifierPath + ".data.lo.contents", m.lo.contents.size(), "\"contents\""); }, [&](ShapeSys const &m) { - modifierPath += "(" + m.name + ")"; checkUniqueName(modifierPath, "shapesys", m.name); if (!m.vals.empty()) { checkSize(modifierPath + ".data.vals", m.vals.size(), "\"vals\""); @@ -691,7 +686,6 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const checkGammaConstraint(modifierPath, m.constraint); }, [&](StatError const &m) { - modifierPath += "(" + m.name + ")"; if (!m.parameters.empty()) { checkSize(modifierPath + ".parameters", m.parameters.size(), "\"parameters\""); } @@ -699,14 +693,13 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const checkStatErrorConstraints(modifierPath, m.constraint); }, [&](CustomModifier const &m) { - modifierPath += "(" + m.name + ")"; if (!isDefinedIn(m.name, "functions") && !isDefinedIn(m.name, "distributions") && !isExternal(m.name)) { error(modifierPath, "unable to find custom modifier '" + m.name + "'"); } }, }, - sample.modifiers[iModifier]); + modifier); } } } diff --git a/roofit/hs3/src/HS3Variates.cxx b/roofit/hs3/src/HS3Variates.cxx new file mode 100644 index 0000000000000..9af3005cf6a7a --- /dev/null +++ b/roofit/hs3/src/HS3Variates.cxx @@ -0,0 +1,485 @@ +/* + * Project: RooFit + * Authors: + * Carsten D. Burgard, DESY/ATLAS, Nov 2026 + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// HS3 requires the terms of a likelihood, and the components of a combined distribution or data set, to have distinct +// variates: a variate in two of them is one quantity modelled twice (e.g. a constraint that is a term of the likelihood +// and a factor of a channel), or two quantities with one name (e.g. the observable "mass" of two channels, which need +// their own names, "mass_SR" and "mass_CR"). All cases are checked with the same function, sharedVariates(), on the +// document, when it is exported and when it is imported. The generic distributions are read as far as the standard +// says what they consist of, through the nodes that the document refers to. + +#include "HS3Document.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { +namespace { + +// Append a name to a list that has every name once. +void addUnique(std::vector &names, std::string const &name) +{ + if (std::find(names.begin(), names.end(), name) == names.end()) { + names.push_back(name); + } +} + +bool isName(JSONNode const &node) +{ + return !node.is_container() && !node.is_null() && !node.is_number(); +} + +// Add the names in `node` (one name or a list) to `out`. +void addNames(JSONNode const &node, std::set &out) +{ + if (node.is_seq()) { + for (JSONNode const &child : node.children()) { + addNames(child, out); + } + } else if (isName(node)) { + out.insert(node.val()); + } +} + +// The names in the list `key` of a generic distribution. The entries of the list are the names, or components that +// give the name as `field`. +std::vector namesIn(JSONNode const &node, const char *key, const char *field = nullptr) +{ + std::vector out; + if (JSONNode const *list = node.find(key)) { + for (JSONNode const &entry : list->children()) { + JSONNode const *name = field ? entry.find(field) : &entry; + if (name && isName(*name)) { + out.push_back(name->val()); + } + } + } + return out; +} + +// The constraint terms that the modifiers of a HistFactory distribution name in "constraints". The "constraint_name" +// of older documents is not counted: it predates the requirement of distinct variates. +std::vector namedConstraints(HistFactoryDist const &dist) +{ + std::vector out; + auto add = overloaded{[&](ConstraintPdf const &constraint) { + if (!constraint.legacyName) { + out.push_back(constraint.pdf.name); + } + }, + [&](GammaConstraintPdfs const &constraints) { + for (auto const &pdf : constraints.pdfs) { + if (pdf) { + out.push_back(pdf->name); + } + } + }, + [](auto const &) {}}; + for (Sample const &sample : dist.samples) { + for (Modifier const &modifier : sample.modifiers) { + std::visit(overloaded{[](CustomModifier const &) {}, [&](auto const &m) { std::visit(add, m.constraint); }}, + modifier); + } + } + return out; +} + +// The same for a HistFactory distribution that the document doesn't interpret, because a user registered their own +// importer or exporter for it. +std::vector namedConstraints(JSONNode const &dist) +{ + std::vector out; + if (JSONNode const *samples = dist.find("samples")) { + for (JSONNode const &sample : samples->children()) { + if (JSONNode const *modifiers = sample.find("modifiers")) { + for (JSONNode const &modifier : modifiers->children()) { + for (std::string const &constraint : namesIn(modifier, "constraints")) { + out.push_back(constraint); + } + } + } + } + } + return out; +} + +// The node of a distribution that the document doesn't interpret, if it has one and, optionally, the type `type`. +JSONNode const *genericNode(Distribution const &dist, const char *type = nullptr) +{ + auto const *generic = std::get_if(&dist.body); + return generic && (!type || generic->type == type) ? generic->node : nullptr; +} + +// The distributions of a document by name, and what they say about the variates. The document must outlive this +// object. +class Distributions { +public: + explicit Distributions(HS3Document const &doc) : _doc{doc} + { + for (Distribution const &dist : doc.distributions) { + _byName.emplace(dist.name, &dist); + } + } + + // The distribution `name` if the document represents it as a `Body_t`, or nullptr. + template + Body_t const *find(std::string const &name) const + { + auto found = _byName.find(name); + return found != _byName.end() ? std::get_if(&found->second->body) : nullptr; + } + + // The node of the generic distribution `name` if it has the type `type`, or nullptr. + JSONNode const *nodeOf(std::string const &name, const char *type) const + { + auto found = _byName.find(name); + return found != _byName.end() ? genericNode(*found->second, type) : nullptr; + } + + // Add the variates of the distribution `name` to `out`: the "x" of a fundamental distribution (a name or a list) + // and the axes of a distribution with axes (e.g. histfactory_dist), through products, mixtures and combined + // distributions, and the "x" of the components of an array distribution. The constraint terms named by a + // histfactory_dist are terms of their own, see likelihoodTerms(). Distributions in `visited` are skipped. + void addVariates(std::string const &name, std::set &out, std::set &visited) const + { + if (!visited.insert(name).second) { + return; + } + if (auto const *histFactory = find(name)) { + for (BinnedAxis const &axis : histFactory->axes) { + out.insert(axis.name); + } + } + auto const *generic = find(name); + if (!generic || !generic->node) { + return; + } + JSONNode const &node = *generic->node; + auto addVariatesOf = [&](std::vector const &parts) { + for (std::string const &part : parts) { + addVariates(part, out, visited); + } + }; + if (generic->type == "product_dist") { + addVariatesOf(namesIn(node, "factors")); + } else if (generic->type == "mixture_dist") { + addVariatesOf(namesIn(node, "summands")); + } else if (generic->type == "combined_dist") { + addVariatesOf(namesIn(node, "components", "dist")); + } else if (generic->type == "array_dist") { + if (JSONNode const *components = node.find("components")) { + for (JSONNode const &component : components->children()) { + if (JSONNode const *x = component.find("x")) { + addNames(*x, out); + } + } + } + } else { + if (JSONNode const *x = node.find("x")) { + addNames(*x, out); + } + for (std::string const &axis : namesIn(node, "axes", "name")) { + out.insert(axis); + } + } + } + + // The terms of a likelihood, each once: its distributions (the components of a combined distribution, or of a + // RooSimultaneous written to ROOT_internal by older versions, are terms of their own), its "aux_distributions", + // and the constraint terms named by the modifiers of its histfactory_dist terms. + std::vector likelihoodTerms(Likelihood const &likelihood) const + { + std::vector terms; + auto add = [&](std::string const &name) { + auto legacy = _doc.rooFit.combinedDistributions.find(name); + if (JSONNode const *combined = nodeOf(name, "combined_dist")) { + for (std::string const &component : namesIn(*combined, "components", "dist")) { + addUnique(terms, component); + } + } else if (!_byName.count(name) && legacy != _doc.rooFit.combinedDistributions.end()) { + for (DistributionRef const &component : legacy->second.distributions) { + addUnique(terms, component.name); + } + } else { + addUnique(terms, name); + } + }; + for (LikelihoodTerm const &term : likelihood.terms) { + add(term.distribution.name); + } + for (DistributionRef const &aux : likelihood.auxDistributions) { + add(aux.name); + } + // The list grows by the constraint terms while it is read. + for (std::size_t i = 0; i < terms.size(); ++i) { + std::vector constraints; + if (auto const *histFactory = find(terms[i])) { + constraints = namedConstraints(*histFactory); + } else if (JSONNode const *generic = nodeOf(terms[i], "histfactory_dist")) { + constraints = namedConstraints(*generic); + } + for (std::string const &constraint : constraints) { + addUnique(terms, constraint); + } + } + return terms; + } + +private: + HS3Document const &_doc; + std::map _byName; +}; + +// The parts (terms or components) of something that must have distinct variates, with their variates. +using PartVariates = std::vector>>; + +// Append a message for each variate that several of `parts` share. `what` names the parts and `context` the whole, +// e.g. "terms" of "the likelihood 'lh'". +void addShared(PartVariates const &parts, std::string const &what, std::string const &context, + std::vector &messages) +{ + std::map> partsOfVariate; + std::vector order; + for (auto const &[part, variates] : parts) { + for (std::string const &variate : variates) { + auto &list = partsOfVariate[variate]; + if (list.empty()) { + order.push_back(variate); + } + addUnique(list, part); + } + } + for (std::string const &variate : order) { + auto const &list = partsOfVariate[variate]; + if (list.size() < 2) { + continue; + } + std::string names; + for (std::size_t i = 0; i < list.size(); ++i) { + names += (i == 0 ? "'" : (i + 1 == list.size() ? "' and '" : "', '")) + list[i]; + } + messages.push_back("the variate '" + variate + "' is shared by the " + what + " " + names + "' of " + context); + } +} + +// The axes of the components of a combined data set, by component. +PartVariates componentAxes(HS3Document const &doc, CombinedData const &combined) +{ + PartVariates parts; + for (CombinedDataComponent const &component : combined.components) { + std::set axes; + auto found = std::find_if(doc.data.begin(), doc.data.end(), + [&](Data const &data) { return data.name == component.data.name; }); + if (found != doc.data.end()) { + std::visit(overloaded{[](CombinedData const &) {}, + [&](PointData const &data) { + for (ContinuousAxis const &axis : data.axes) { + axes.insert(axis.name); + } + }, + [&](auto const &data) { + for (auto const &axis : data.axes) { + axes.insert(nameOf(axis)); + } + }}, + found->body); + } + parts.emplace_back(component.data.name, axes); + } + return parts; +} + +} // namespace + +std::set variatesOf(HS3Document const &doc, Likelihood const &likelihood) +{ + Distributions distributions{doc}; + std::set variates; + std::set visited; + for (LikelihoodTerm const &term : likelihood.terms) { + distributions.addVariates(term.distribution.name, variates, visited); + if (JSONNode const *combined = distributions.nodeOf(term.distribution.name, "combined_dist")) { + for (std::string const &column : namesIn(*combined, "columns")) { + variates.insert(column); + } + } + } + return variates; +} + +std::map> sharedAxes(HS3Document const &doc) +{ + std::map> out; + for (Data const &data : doc.data) { + if (auto const *combined = std::get_if(&data.body)) { + std::map counts; + for (auto const &[component, axes] : componentAxes(doc, *combined)) { + for (std::string const &axis : axes) { + if (++counts[axis] == 2) { + out[data.name].insert(axis); + } + } + } + } + } + return out; +} + +std::map> bookkeepingColumns(HS3Document const &doc) +{ + std::map> out; + if (doc.rooFit.bookkeepingColumns.empty()) { + return out; + } + Distributions distributions{doc}; + std::set variates; + std::set visited; + for (Distribution const &dist : doc.distributions) { + distributions.addVariates(dist.name, variates, visited); + } + for (auto const &[dataset, columns] : doc.rooFit.bookkeepingColumns) { + for (std::string const &column : columns) { + if (!variates.count(column)) { + out[dataset].insert(column); + } + } + } + return out; +} + +std::vector sharedVariates(HS3Document const &doc) +{ + Distributions distributions{doc}; + std::vector messages; + auto withVariates = [&](std::vector const &names) { + PartVariates parts; + for (std::string const &name : names) { + std::set visited; + distributions.addVariates(name, parts.emplace_back(name, std::set{}).second, visited); + } + return parts; + }; + std::set inLikelihoods; + for (Likelihood const &likelihood : doc.likelihoods) { + for (LikelihoodTerm const &term : likelihood.terms) { + inLikelihoods.insert(term.distribution.name); + } + for (DistributionRef const &aux : likelihood.auxDistributions) { + inLikelihoods.insert(aux.name); + } + addShared(withVariates(distributions.likelihoodTerms(likelihood)), "terms", + "the likelihood '" + likelihood.name + "'", messages); + } + for (Distribution const &dist : doc.distributions) { + JSONNode const *combined = genericNode(dist, "combined_dist"); + if (combined && !inLikelihoods.count(dist.name)) { + addShared(withVariates(namesIn(*combined, "components", "dist")), "components", + "the combined_dist '" + dist.name + "'", messages); + } + } + // The axes of the components of every combined data set, also of those that a likelihood lists. + std::map> const bookkeeping = bookkeepingColumns(doc); + for (Data const &data : doc.data) { + auto const *combined = std::get_if(&data.body); + if (!combined) { + continue; + } + PartVariates parts = componentAxes(doc, *combined); + if (auto found = bookkeeping.find(data.name); found != bookkeeping.end()) { + for (auto &part : parts) { + for (std::string const &column : found->second) { + part.second.erase(column); + } + } + } + addShared(parts, "components", "the combined data set '" + data.name + "'", messages); + } + return messages; +} + +std::map> writeLikelihoodTermsOnce(HS3Document const &doc, JSONNode &root) +{ + std::map> removed; + if (!root.has_child("distributions")) { + return removed; + } + Distributions distributions{doc}; + // The products that are terms, with the factors that are terms wherever the product is one. + std::map> removable; + for (Likelihood const &likelihood : doc.likelihoods) { + std::vector const terms = distributions.likelihoodTerms(likelihood); + for (std::string const &term : terms) { + JSONNode const *product = distributions.nodeOf(term, "product_dist"); + if (!product) { + continue; + } + std::set factorsThatAreTerms; + for (std::string const &factor : namesIn(*product, "factors")) { + if (factor != term && std::find(terms.begin(), terms.end(), factor) != terms.end()) { + factorsThatAreTerms.insert(factor); + } + } + auto found = removable.find(term); + if (found == removable.end()) { + removable[term] = factorsThatAreTerms; + } else { + std::set both; + std::set_intersection(found->second.begin(), found->second.end(), factorsThatAreTerms.begin(), + factorsThatAreTerms.end(), std::inserter(both, both.begin())); + found->second = both; + } + } + } + // Products that another distribution uses are left alone. + for (Distribution const &dist : doc.distributions) { + if (JSONNode const *node = genericNode(dist)) { + for (const char *key : {"factors", "summands"}) { + for (std::string const &part : namesIn(*node, key)) { + removable.erase(part); + } + } + } + } + for (JSONNode &dist : root["distributions"].children()) { + auto found = removable.find(dist["name"].val()); + if (found == removable.end() || found->second.empty()) { + continue; + } + std::vector keep; + for (JSONNode const &factor : dist["factors"].children()) { + if (!found->second.count(factor.val())) { + keep.push_back(factor.val()); + } + } + if (!keep.empty()) { + dist["factors"].clear(); + dist["factors"].fill_seq(keep); + removed.insert(*found); + } + } + return removed; +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Writer.cxx b/roofit/hs3/src/HS3Writer.cxx index 9498fbde8fba8..b8027150f7c50 100644 --- a/roofit/hs3/src/HS3Writer.cxx +++ b/roofit/hs3/src/HS3Writer.cxx @@ -14,7 +14,6 @@ // tests in testHS3Document.cxx check. #include "HS3Document.h" -#include "JSONIOUtils.h" #include @@ -38,11 +37,6 @@ void writeBound(JSONNode &node, double value) } } -void writeStrings(JSONNode &node, std::vector const &values) -{ - node.fill_seq(values); -} - template void writeRefs(JSONNode &node, std::vector> const &refs) { @@ -68,18 +62,6 @@ void writeBinning(JSONNode &node, Binning const &binning) } } -void writeAxis(JSONNode &node, BinnedAxis const &axis) -{ - node.set_map(); - node["name"] << axis.name; - // The range of an irregular binning is given by its "edges": HS3 0.3 forbids "min" and "max" alongside them. - if (!std::holds_alternative(axis.binning)) { - node["min"] << axis.min; - node["max"] << axis.max; - } - writeBinning(node, axis.binning); -} - void writeAxis(JSONNode &node, ContinuousAxis const &axis) { node.set_map(); @@ -96,7 +78,7 @@ void writeAxis(JSONNode &node, CategoricalAxis const &axis) { node.set_map(); node["name"] << axis.name; - writeStrings(node["categories"], axis.categories); + node["categories"].fill_seq(axis.categories); } // An infinite or missing bound of a domain axis is omitted, which means that the range is unbounded. @@ -105,7 +87,7 @@ void writeAxis(JSONNode &node, DomainAxis const &axis) node.set_map(); node["name"] << axis.name; if (axis.fixedValue) { - node[fixedValueKey] << *axis.fixedValue; + node["value"] << *axis.fixedValue; return; } // HS3 0.3 forbids "min" and "max" alongside "edges", which give the range. They are only written if they differ from @@ -244,13 +226,7 @@ void writeData(JSONNode &dataArray, Data const &data) std::visit(overloaded{ [&](BinnedData const &d) { node["type"] << "binned"; - writeAxes(node["axes"], d.axes); - writeContents(node["contents"], d.contents); - if (d.uncertainty) { - JSONNode &uncertainty = node["uncertainty"].set_map(); - uncertainty["type"] << "gaussian_uncertainty"; - uncertainty["sigma"].fill_seq(d.uncertainty->sigma); - } + writeBinnedData(node, d); }, [&](UnbinnedData const &d) { node["type"] << "unbinned"; @@ -274,11 +250,11 @@ void writeData(JSONNode &dataArray, Data const &data) }, [&](CombinedData const &d) { node["type"] << "combined"; - writeStrings(node["columns"], d.columns); + node["columns"].fill_seq(d.columns); JSONNode &components = node["components"].set_seq(); for (CombinedDataComponent const &component : d.components) { JSONNode &out = components.append_child().set_map(); - writeStrings(out["key"], component.key); + out["key"].fill_seq(component.key); out["dataset"] << component.data.name; } }, @@ -342,7 +318,7 @@ void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) } } if (!item.second.tags.empty()) { - writeStrings(node["tags"], item.second.tags); + node["tags"].fill_seq(item.second.tags); } } } @@ -351,7 +327,7 @@ void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) for (auto const &item : internal.combinedDistributions) { JSONNode &node = combined[item.first].set_map(); node["index_cat"] << item.second.indexCat; - writeStrings(node["labels"], item.second.labels); + node["labels"].fill_seq(item.second.labels); node["indices"].fill_seq(item.second.indices); writeRefs(node["distributions"], item.second.distributions); } @@ -361,7 +337,7 @@ void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) for (auto const &item : internal.combinedDatasets) { JSONNode &node = combined[item.first].set_map(); node["index_cat"] << item.second.indexCat; - writeStrings(node["labels"], item.second.labels); + node["labels"].fill_seq(item.second.labels); node["indices"].fill_seq(item.second.indices); } } @@ -392,7 +368,7 @@ void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) if (!internal.bookkeepingColumns.empty()) { JSONNode &columns = rootInternal()["bookkeeping_columns"].set_map(); for (auto const &item : internal.bookkeepingColumns) { - writeStrings(columns[item.first], item.second); + columns[item.first].fill_seq(item.second); } } if (!internal.stepWidths.empty()) { @@ -477,6 +453,29 @@ void writeInterpolated(JSONNode &node, Function_t const &function, char const *t } // namespace +void writeAxis(JSONNode &node, BinnedAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + // The range of an irregular binning is given by its "edges": HS3 0.3 forbids "min" and "max" alongside them. + if (!std::holds_alternative(axis.binning)) { + node["min"] << axis.min; + node["max"] << axis.max; + } + writeBinning(node, axis.binning); +} + +void writeBinnedData(JSONNode &node, BinnedData const &data) +{ + writeAxes(node["axes"], data.axes); + writeContents(node["contents"], data.contents); + if (data.uncertainty) { + JSONNode &uncertainty = node["uncertainty"].set_map(); + uncertainty["type"] << "gaussian_uncertainty"; + uncertainty["sigma"].fill_seq(data.uncertainty->sigma); + } +} + void writeInterpolatedFunction(JSONNode &node, InterpolatedFunction const &function) { writeInterpolated(node, function, "interpolation"); diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 6281f2f64ffd5..932f5d2d6e1fe 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -36,7 +36,6 @@ #include #include -#include #include #include #include @@ -52,11 +51,9 @@ #include "JSONIOUtils.h" using RooFit::Detail::JSONNode; -using RooFit::HS3::additivePolynomialLinear; using RooFit::HS3::Interpolation; using RooFit::HS3::InterpolationClass; using RooFit::HS3::multiplicativePiecewiseExponential; -using RooFit::HS3::multiplicativePolynomialExponential; namespace HS3 = RooFit::HS3; @@ -77,16 +74,13 @@ double round_prec(double d, int nSig) return sf * std::round(d / sf); } +// An interpolation as it is written in the document, for messages. std::string interpolationString(const Interpolation &interpolation) { + auto tree = RooFit::Detail::JSONTree::create(); + HS3::writeInterpolation(tree->rootnode(), interpolation); std::stringstream ss; - ss << R"({"type":")" << toString(interpolation.type) << R"(","in":")" << toString(interpolation.in) << R"(","out":)"; - if (interpolation.out) { - ss << '"' << toString(*interpolation.out) << '"'; - } else { - ss << "null"; - } - ss << '}'; + tree->rootnode().writeJSON(ss); return ss.str(); } @@ -139,29 +133,6 @@ std::vector toCodes(std::vector const &interpolations, Inter return out; } -// Reports the problems with an object that doesn't come with a document. -void report(std::string const &name, HS3::Diagnostics &diagnostics) -{ - for (auto &diagnostic : diagnostics) { - diagnostic.path = name + (diagnostic.path.empty() ? "" : "." + diagnostic.path); - } - HS3::report(diagnostics); -} - -int interpolationCode(std::optional const &interpolation, - std::optional const &defaultInterpolation, InterpolationClass interpolationClass, - std::string const &context) -{ - if (interpolation) { - return codeFromInterpolation(*interpolation, interpolationClass, context); - } - // HS3 forbids implementation-specific default interpolations. - if (!defaultInterpolation) { - RooJSONFactoryWSTool::error(context + " is not given, and the channel has no \"default_interpolation\""); - } - return codeFromInterpolation(*defaultInterpolation, interpolationClass, context); -} - void erasePrefix(std::string &str, std::string_view prefix) { if (startsWith(str, prefix)) { @@ -185,16 +156,22 @@ void sortByName(Coll &coll) std::sort(coll.begin(), coll.end(), [](auto &l, auto &r) { return l.name < r.name; }); } +// Whether any of the gamma parameters names a constraint. Otherwise, they are all free, and nothing is written. +bool anyNamed(HS3::GammaConstraintPdfs const &constraints) +{ + return std::any_of(constraints.pdfs.begin(), constraints.pdfs.end(), + [](auto const &pdf) { return pdf.has_value(); }); +} + template T *findClient(RooAbsArg *gamma) { for (const auto &client : gamma->clients()) { if (auto casted = dynamic_cast(client)) { return casted; - } else { - T *c = findClient(client); - if (c) - return c; + } + if (T *found = findClient(client)) { + return found; } } return nullptr; @@ -231,11 +208,6 @@ RooAbsPdf *findConstraint(RooAbsArg *g, std::vector const &channelC return findClient(g); } -inline std::string defaultGammaName(std::string const &sysname, std::size_t i) -{ - return "gamma_" + sysname + "_bin_" + std::to_string(i); -} - RooRealVar &createNominal(RooWorkspace &ws, std::string const &parname, double val, double min, double max) { RooRealVar &nom = getOrCreate(ws, "nom_" + parname, val, min, max); @@ -274,20 +246,16 @@ RooAbsPdf &arrayComponentFor(RooProdPdf const &array, RooAbsArg const ¶m, st // Find the constraint distribution `constraintName` of the parameter `param`. If the name resolves to an array // distribution, the component that depends on the parameter is returned: that is the term that enters the likelihood -// for this parameter, and the whole array enters once through the components of all of its parameters. Returns -// nullptr if the name cannot be resolved. -RooAbsPdf *findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &constraintName, RooAbsArg const ¶m, +// for this parameter, and the whole array enters once through the components of all of its parameters. +RooAbsPdf &findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &constraintName, RooAbsArg const ¶m, std::string const &context) { RooAbsPdf *constraint = tool.tryRequest(constraintName); - if (!constraint || !constraint->getAttribute(arrayDistTag)) { - return constraint; + if (!constraint) { + RooJSONFactoryWSTool::error("unable to find the constraint '" + constraintName + "' of " + context); } auto const *array = dynamic_cast(constraint); - if (!array) { - return constraint; - } - return &arrayComponentFor(*array, param, context); + return array && array->getAttribute(arrayDistTag) ? arrayComponentFor(*array, param, context) : *constraint; } // The parameter of a modifier, which is created if the document doesn't define it. It's a variable, or a constant if @@ -337,17 +305,13 @@ void addConstraint(RooJSONFactoryWSTool &tool, HS3::Constraint const &constraint constraints.add(pdf ? *pdf : tool.wsEmplace(name, param, nom, 1.)); }, [&](HS3::ConstraintPdf const &constraintPdf) { - RooAbsPdf *pdf = findNamedConstraint(tool, constraintPdf.pdf.name, param, + RooAbsPdf &pdf = findNamedConstraint(tool, constraintPdf.pdf.name, param, std::string("the ") + modifierType + " modifier '" + modifier + "' of sample '" + sample + "'"); - if (!pdf) { - RooJSONFactoryWSTool::error("unable to find definition of of constraint '" + - constraintPdf.pdf.name + "' for modifier '" + modifier + "'"); - } - if (auto gauss = dynamic_cast(pdf)) { + if (auto gauss = dynamic_cast(&pdf)) { setErrorIfVariable(param, gauss->getSigma().getVal()); } - constraints.add(*pdf); + constraints.add(pdf); }, }, constraint); @@ -376,24 +340,13 @@ std::optional findPoissonTau(RooPoisson const &constraint, RooAbsArg con return std::nullopt; } + // The tau is the constant factor. In imported workspaces, it can also be a variable that is named like it. for (RooAbsArg *arg : mean->servers()) { - if (arg == &gamma) { - continue; - } - - if (auto const *tau = dynamic_cast(arg)) { + auto const *tau = dynamic_cast(arg); + if (tau && arg != &gamma && (tau->isConstant() || endsWith(tau->GetName(), "_tau"))) { return tau->getVal(); } - - // Imported workspaces can sometimes represent - // constants as constant RooRealVars. - if (auto const *real = dynamic_cast(arg)) { - if (real->isConstant() || endsWith(std::string(real->GetName()), "_tau")) { - return real->getVal(); - } - } } - return std::nullopt; } @@ -458,17 +411,14 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, // The validation made sure that there is a value, parameter and constraint for each bin, if they are given. RooArgList gammas; for (std::size_t i = 0; i < nBins; ++i) { - const std::string name = parnames.empty() ? defaultGammaName(sysname, i) : parnames[i].name; - if (auto *existing = dynamic_cast(ws.obj(name))) { - gammas.add(*existing); - } else { - gammas.add(getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); - } + const std::string name = parnames.empty() ? "gamma_" + sysname + "_bin_" + std::to_string(i) : parnames[i].name; + auto *existing = dynamic_cast(ws.obj(name)); + gammas.add(existing ? *existing : getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); } // The gammas and `vals` are given in the order of HS3 (last axis fastest), which is also the order in which they // are paired below. ParamHistFunc expects its parameters in TH1 order. - checkParamHistFuncOrderIsDefined(tool.hs3VersionInput(), observables, "the modifier '" + sysname + "'"); + checkParamHistFuncOrderIsDefined(tool.hs3Version(), observables, "the modifier '" + sysname + "'"); RooArgList gammasInTH1Order = toParamHistFuncOrder(gammas, observables); // Shape modifiers of the same name in a channel share the function, so they need the same parameters. @@ -487,8 +437,7 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, auto addGammaConstraints = [&](Constraint::Type type) { for (auto const &term : createGammaConstraints(gammas, vals, minSigma, type).constraints) { - ws.import(*term, RooFit::RecycleConflictNodes(), RooFit::Silence()); - constraints.add(*ws.pdf(term->GetName())); + constraints.add(tool.wsImport(*term)); } }; @@ -498,13 +447,8 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, std::vector constraintPdfs; for (std::size_t i = 0; i < pdfs->pdfs.size(); ++i) { auto const &pdfName = pdfs->pdfs[i]; - RooAbsPdf *pdf = nullptr; - if (pdfName) { - pdf = findNamedConstraint(tool, pdfName->name, gammas[i], context); - if (!pdf) { - RooJSONFactoryWSTool::error("unable to find definition of constraint '" + pdfName->name + - "' for modifier '" + sysname + "'"); - } + RooAbsPdf *pdf = pdfName ? &findNamedConstraint(tool, pdfName->name, gammas[i], context) : nullptr; + if (pdf) { constraints.add(*pdf); } constraintPdfs.push_back(pdf); @@ -555,7 +499,6 @@ double constraintRelError(RooAbsPdf const &constraint, RooAbsArg const &gamma) gamma.GetName() + "': only RooGaussian and RooPoisson with mean gamma * tau are supported for the " "relative uncertainties of gammas"); - return std::numeric_limits::quiet_NaN(); } struct ChannelContext { @@ -564,7 +507,6 @@ struct ChannelContext { RooArgSet const &observables; std::string fprefix; std::string channelName; - std::size_t nBins; RooAbsArg const *mcStatObject; RooArgSet &constraints; }; @@ -614,17 +556,19 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi elems.shapeElems.add(*ctx.mcStatObject); } - auto context = [&](const char *type, std::string const &name) { - return std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + sampleName + - "' of channel '" + ctx.channelName + "'"; + // The validation made sure that the channel has a "default_interpolation" if a modifier doesn't give its own. + auto interpolationCode = [&](std::optional const &interpolation, + InterpolationClass interpolationClass, const char *type, std::string const &name) { + return codeFromInterpolation(interpolation ? *interpolation : ctx.dist.defaultInterpolation.value(), + interpolationClass, + std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + + sampleName + "' of channel '" + ctx.channelName + "'"); }; for (HS3::Modifier const &modifier : sample.modifiers) { std::visit( HS3::overloaded{ - [&](HS3::StatError const &) { - // this is dealt with at a different place, ignore it for now - }, + [](HS3::StatError const &) {}, // built once for the channel, in buildHistFactoryDist() [&](HS3::NormFactor const &mod) { RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 1., -3, 5, mod.name); // Norm factors get an initial zero error like in HistoToWorkspaceFactoryFast, unless an error was @@ -633,28 +577,20 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi var->setError(0.0); } elems.normElems.add(param); - // for norm factors, constraints are optional addConstraint(tool, mod.constraint, param, 1.0, "normfactor", mod.name, sampleName, ctx.constraints); }, [&](HS3::NormSys const &mod) { RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); elems.overallNps.add(param); - const int interp = interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, - InterpolationClass::Flexible, context("normsys", mod.name)); - double low = mod.lo; - double high = mod.hi; - - // the below contains a a hack to cut off variations that go below 0 - // This is needed because FlexibleInterpVar code 4 interpolates in log-space. Hence, values <= 0 result - // in NaN, which propagates throughout the model and causes evaluations to fail. If you know a nicer way - // to solve this, please go ahead and fix the lines below. - if (interp == 4 && low <= 0) - low = std::numeric_limits::epsilon(); - if (interp == 4 && high <= 0) - high = std::numeric_limits::epsilon(); - - elems.overallLow.push_back(low); - elems.overallHigh.push_back(high); + const int interp = + interpolationCode(mod.interpolation, InterpolationClass::Flexible, "normsys", mod.name); + // FlexibleInterpVar code 4 interpolates in log-space, so variations <= 0 would give NaN, which + // propagates through the model and makes its evaluation fail. As a hack, they are cut off. + auto cutOff = [&](double variation) { + return interp == 4 && variation <= 0 ? std::numeric_limits::epsilon() : variation; + }; + elems.overallLow.push_back(cutOff(mod.lo)); + elems.overallHigh.push_back(cutOff(mod.hi)); elems.overallInterp.push_back(interp); addSysConstraint(tool, mod.constraint, param, "normsys", mod.name, sampleName, ctx.constraints); @@ -671,19 +607,15 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi }; addVariation(elems.histoLo, mod.lo, "Low_"); addVariation(elems.histoHi, mod.hi, "High_"); - elems.histoInterp.push_back(interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, - InterpolationClass::Piecewise, - context("histosys", mod.name))); + elems.histoInterp.push_back( + interpolationCode(mod.interpolation, InterpolationClass::Piecewise, "histosys", mod.name)); addSysConstraint(tool, mod.constraint, param, "histosys", mod.name, sampleName, ctx.constraints); }, [&](HS3::ShapeSys const &mod) { const bool constrained = std::visit( HS3::overloaded{[](HS3::Unconstrained) { return false; }, [](HS3::ConstantGammas) { return false; }, [&](HS3::ConstraintType) { return !mod.vals.empty(); }, - [](HS3::GammaConstraintPdfs const &constraints) { - return std::any_of(constraints.pdfs.begin(), constraints.pdfs.end(), - [](auto const &pdf) { return pdf.has_value(); }); - }}, + [](HS3::GammaConstraintPdfs const &constraints) { return anyNamed(constraints); }}, mod.constraint); // Follow the naming of HistFactory: "__ShapeSys" for constrained and // "__shapeFactor" for free parameters. @@ -691,7 +623,7 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi ctx.channelName + "_" + mod.name + (constrained ? "_ShapeSys" : "_shapeFactor"); elems.shapeElems.add( createPHF(tool, funcName, mod.name, mod.parameters, mod.vals, mod.constraint, ctx.constraints, - ctx.observables, defaultShapeSysGammaMax, minShapeUncertainty, ctx.nBins, + ctx.observables, defaultShapeSysGammaMax, minShapeUncertainty, dh.numEntries(), "the shapesys modifier '" + mod.name + "' of sample '" + sampleName + "'")); }, [&](HS3::CustomModifier const &mod) { @@ -844,7 +776,7 @@ class PiecewiseInterpolationFactory : public RooFit::JSONIO::Importer { std::string name(RooJSONFactoryWSTool::name(p)); HS3::Diagnostics diagnostics; const HS3::InterpolatedFunction in = HS3::parseInterpolatedFunction(p, diagnostics); - report(name, diagnostics); + HS3::report(diagnostics, name); auto request = [&](std::vector const &refs) { RooArgList out; @@ -872,7 +804,7 @@ class FlexibleInterpVarFactory : public RooFit::JSONIO::Importer { std::string name(RooJSONFactoryWSTool::name(p)); HS3::Diagnostics diagnostics; const HS3::InterpolatedNumber in = HS3::parseInterpolatedNumber(p, diagnostics); - report(name, diagnostics); + HS3::report(diagnostics, name); RooArgList vars; for (std::string const &var : in.parameters) { @@ -1000,37 +932,23 @@ FormulaNormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar * FormulaNormSys sys; std::smatch match; - if (std::regex_match(s, match, pattern)) { - const double sign = match[1].str() == "-" ? -1.0 : 1.0; - - std::string token2 = match[2].str(); - std::string token3 = match[4].str(); - - RooAbsReal *p2 = static_cast(formula->getParameter(token2.c_str())); - RooAbsReal *p3 = static_cast(formula->getParameter(token3.c_str())); - RooRealVar *v2 = dynamic_cast(p2); - RooRealVar *v3 = dynamic_cast(p3); - - auto *constr2 = findConstraint(v2, channelConstraints); - auto *constr3 = findConstraint(v3, channelConstraints); - - if (constr2 && !p3) { - sys.param = p2; - sys.variation = sign * toDouble(token3); - } else if (!p2 && constr3) { - sys.param = p3; - sys.variation = sign * toDouble(token2); - } else if (constr2 && p3 && !constr3) { - sys.param = v2; - sys.variation = sign * p3->getVal(); - } else if (p2 && !constr2 && constr3) { - sys.param = v3; - sys.variation = sign * p2->getVal(); - } - - // A recognized formula has exactly one constrained parameter. - sys.constraint = constr2 ? constr2 : constr3; - } + if (!std::regex_match(s, match, pattern)) { + return sys; + } + // The factors a and b are parameters of the formula or numbers. A recognized formula has exactly one constrained + // parameter, and the value of the other factor is the variation. + auto *a = static_cast(formula->getParameter(match[2].str().c_str())); + auto *b = static_cast(formula->getParameter(match[4].str().c_str())); + RooAbsPdf *constraintA = findConstraint(dynamic_cast(a), channelConstraints); + RooAbsPdf *constraintB = findConstraint(dynamic_cast(b), channelConstraints); + if (!constraintA == !constraintB) { + return sys; + } + RooAbsReal *other = constraintA ? b : a; + const double sign = match[1].str() == "-" ? -1.0 : 1.0; + sys.param = constraintA ? a : b; + sys.constraint = constraintA ? constraintA : constraintB; + sys.variation = sign * (other ? other->getVal() : toDouble(match[constraintA ? 4 : 2].str())); return sys; } @@ -1047,12 +965,7 @@ void collectElements(RooArgList &elems, RooAbsArg *arg) bool allRooRealVar(const RooAbsCollection &list) { - for (auto *var : list) { - if (!dynamic_cast(var)) { - return false; - } - } - return true; + return std::all_of(list.begin(), list.end(), [](RooAbsArg *arg) { return dynamic_cast(arg); }); } // The parameters of normsys and histosys modifiers must be variables. @@ -1198,20 +1111,19 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna sumpdf->treeNodeServerList(&channelNodes); const std::vector volumes = binVolumes(*sumpdf); - auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { - if (!pdf) { - return HS3::Unconstrained{}; - } - channel.constraints.emplace(pdf->GetName(), pdf); - return HS3::ConstraintPdf{{pdf->GetName()}}; - }; - auto gammaConstraint = [&](RooAbsPdf const *pdf) -> std::optional { - if (!pdf) { + // The modifiers name only the constraint terms of the channel. A pdf of a parameter that is not, like an external + // constraint or a prior, stays where it is. + auto constraintTerm = [&](RooAbsPdf const *pdf) -> std::optional { + if (std::find(channelConstraints.begin(), channelConstraints.end(), pdf) == channelConstraints.end()) { return std::nullopt; } channel.constraints.emplace(pdf->GetName(), pdf); return HS3::DistributionRef{pdf->GetName()}; }; + auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { + auto ref = constraintTerm(pdf); + return ref ? HS3::Constraint{HS3::ConstraintPdf{*ref}} : HS3::Unconstrained{}; + }; for (size_t sampleidx = 0; sampleidx < sumpdf->funcList().size(); ++sampleidx) { PiecewiseInterpolation *pip = nullptr; @@ -1254,12 +1166,7 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna } else { // The bin contents of all samples are written in the order of the axes of the channel (the last axis // varies fastest), which is the order of the observables of the first histogram. - RooArgSet const &vars = *varsPtr; - bool sameOrder = vars.size() == channel.observables->size(); - for (std::size_t i = 0; sameOrder && i < vars.size(); ++i) { - sameOrder = std::string(vars[i]->GetName()) == (*channel.observables)[i]->GetName(); - } - if (!sameOrder) { + if (varsPtr->size() != channel.observables->size() || !varsPtr->hasSameLayout(*channel.observables)) { RooJSONFactoryWSTool::error("the histograms of channel '" + channel.name + "' don't have their observables in the same order, so their bin " "contents can't be written along the same axes"); @@ -1308,7 +1215,7 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna erasePrefix(sysname, "alpha_"); const std::string context = "normsys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; - // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). + // A parameter without constraint is unconstrained. addNormSys(*var, fip->high()[i], fip->low()[i], interpolationFromCode(fip->interpolationCodes()[i], InterpolationClass::Flexible, context), findConstraint(var, channelConstraints)); @@ -1370,39 +1277,37 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna collectElements(elems, sumpdf->coefList().at(sampleidx)); processElements(elems, processElements); - // see if we can get the observables if (pip) { + // see if we can get the observables if (auto nh = dynamic_cast(pip->nominalHist())) { updateObservables(*nh); } - } - - if (pip) { for (size_t i = 0; i < pip->paramList().size(); ++i) { + auto lo = dynamic_cast(pip->lowList().at(i)); + auto hi = dynamic_cast(pip->highList().at(i)); + if (!lo || !hi) { + continue; + } RooAbsReal *var = static_cast(pip->paramList().at(i)); std::string sysname(var->GetName()); erasePrefix(sysname, "alpha_"); - if (auto lo = dynamic_cast(pip->lowList().at(i))) { - if (auto hi = dynamic_cast(pip->highList().at(i))) { - // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). - const auto *pdf = findConstraint(var, channelConstraints); - checkModifierParameter(*var, pdf); - const std::string context = "histosys modifier '" + sysname + "' in sample '" + sample.name + - "' of channel '" + channel.name + "'"; - histoSys.push_back( - {sysname, - {var->GetName()}, - {contents(hi->dataHist()), std::nullopt}, - {contents(lo->dataHist()), std::nullopt}, - constraint(pdf), - interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context)}); - } - } + // A parameter without constraint is unconstrained. + const auto *pdf = findConstraint(var, channelConstraints); + checkModifierParameter(*var, pdf); + const std::string context = + "histosys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; + histoSys.push_back( + {sysname, + {var->GetName()}, + {contents(hi->dataHist()), std::nullopt}, + {contents(lo->dataHist()), std::nullopt}, + constraint(pdf), + interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context)}); } } for (ParamHistFunc *phf : phfs) { - if (startsWith(std::string(phf->GetName()), "mc_stat_")) { // MC stat uncertainty + if (startsWith(phf->GetName(), "mc_stat_")) { // MC stat uncertainty statError.emplace(); statError->name = "staterror"; HS3::GammaConstraintPdfs pdfs; @@ -1412,14 +1317,16 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna statError->parameters.push_back({g->GetName()}); ++idx; RooAbsPdf *pdf = findConstraint(g, channelConstraints); - pdfs.pdfs.push_back(gammaConstraint(pdf)); + pdfs.pdfs.push_back(constraintTerm(pdf)); channel.totYield[idx] += sample.contents[idx - 1]; channel.totYield2[idx] += (sample.contents[idx - 1] * sample.contents[idx - 1]); if (pdf) { channel.relErrors[idx] = constraintRelError(*pdf, *g); } } - statError->constraint = std::move(pdfs); + if (anyNamed(pdfs)) { + statError->constraint = std::move(pdfs); + } } else { // ShapeSys or ShapeFactor // Other ParamHistFuncs can't be written as shapefactors: the import would recreate the function over the // observables and bins of the channel. Over other variables, they are written as custom modifiers, which @@ -1451,9 +1358,11 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna RooAbsPdf *pdf = findConstraint(g, channelConstraints); if (!pdf) pdf = ws->pdf(constraintName(g->GetName())); - pdfs.pdfs.push_back(gammaConstraint(pdf)); + pdfs.pdfs.push_back(constraintTerm(pdf)); + } + if (anyNamed(pdfs)) { + sys.constraint = std::move(pdfs); } - sys.constraint = std::move(pdfs); } } @@ -1503,21 +1412,24 @@ void configureStatError(ExportedChannel &channel) } } -// Whether `pdf` has the form that the HistFactory exporter writes as a histfactory_dist with its constraint terms: -// a product of exactly one RooRealSumPdf and other pdfs. -bool isHistFactoryChannelProduct(RooAbsArg const &pdf) +// The RooRealSumPdf of a product of exactly one RooRealSumPdf and other pdfs, or null for anything else. Such a product +// has the form that the HistFactory exporter writes as a histfactory_dist with its constraint terms. +RooRealSumPdf const *channelSum(RooAbsArg const &pdf) { auto const *prod = dynamic_cast(&pdf); if (!prod) { - return false; + return nullptr; } - int nSums = 0; + RooRealSumPdf const *sum = nullptr; for (RooAbsArg const *factor : prod->pdfList()) { - if (dynamic_cast(factor)) { - ++nSums; + if (auto const *factorSum = dynamic_cast(factor)) { + if (sum) { + return nullptr; + } + sum = factorSum; } } - return nSums == 1; + return sum; } // Whether the constraint `constraint` can be written as an array component: every object of the workspace that uses @@ -1537,7 +1449,7 @@ bool usedOnlyByHistFactoryChannels(RooJSONFactoryWSTool &tool, RooAbsPdf const & if (tool.workspace()->arg(client->GetName()) != client) { continue; // not part of what is exported } - if (!client->getAttribute(arrayDistTag) && !isHistFactoryChannelProduct(*client)) { + if (!client->getAttribute(arrayDistTag) && !channelSum(*client)) { return false; } } @@ -1627,56 +1539,6 @@ std::vector exportNamedConstraints(RooJSONFactoryWSTool &tool, Expo return terms; } -// The modifiers name only the constraint terms of the channel. A pdf of a parameter that is not, like an external -// constraint or a prior, stays where it is. Returns the named constraint terms. -std::set -nameChannelConstraints(ExportedChannel &channel, std::vector const &constraints) -{ - std::set named; - auto isNamed = [&](std::string const &name) { - RooAbsPdf const *pdf = channel.constraints.at(name); - if (std::find(constraints.begin(), constraints.end(), pdf) == constraints.end()) { - return false; - } - named.insert(pdf); - return true; - }; - auto filter = [&](HS3::Constraint &constraint) { - auto const *pdf = std::get_if(&constraint); - if (pdf && !isNamed(pdf->pdf.name)) { - constraint = HS3::Unconstrained{}; - } - }; - auto filterGammas = [&](HS3::GammaConstraint &constraint) { - auto *pdfs = std::get_if(&constraint); - if (!pdfs) { - return; - } - bool any = false; - for (auto &pdf : pdfs->pdfs) { - if (pdf && !isNamed(pdf->name)) { - pdf.reset(); - } - any = any || pdf; - } - // Nothing is written if all parameters are unconstrained. - if (!any) { - constraint = HS3::Unconstrained{}; - } - }; - for (HS3::Sample &sample : channel.dist.samples) { - for (HS3::Modifier &modifier : sample.modifiers) { - std::visit(HS3::overloaded{ - [&](HS3::NormFactor &m) { filter(m.constraint); }, - [&](HS3::NormSys &m) { filter(m.constraint); }, [&](HS3::HistoSys &m) { filter(m.constraint); }, - [&](HS3::ShapeSys &m) { filterGammas(m.constraint); }, - [&](HS3::StatError &m) { filterGammas(m.constraint); }, [](HS3::CustomModifier &) {}}, - modifier); - } - } - return named; -} - bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf, std::vector constraints) { @@ -1713,7 +1575,6 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname dist.axes.push_back(HS3::toBinnedAxis(*obs)); } - std::set namedConstraints = nameChannelConstraints(channel, constraints); tool->recordConstraintTerms(pdfname, exportNamedConstraints(*tool, channel)); for (RooAbsReal *modifier : channel.customModifiers) { @@ -1728,7 +1589,7 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname sumpdf->getParameters(channel.observables.get(), parameters); for (RooAbsArg *param : parameters) { // This should exclude the global observables. The legacy inverse bin widths are not written at all. - if (!startsWith(std::string{param->GetName()}, "nom_") && !channel.legacyBinWidthFactors.count(param)) { + if (!startsWith(param->GetName(), "nom_") && !channel.legacyBinWidthFactors.count(param)) { tool->queueExport(*param); } } @@ -1736,7 +1597,7 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname // All other constraint terms of the channel are written separately. std::vector channelConstraintNames; for (RooAbsPdf const *pdf : constraints) { - if (!namedConstraints.count(pdf)) { + if (!channel.constraints.count(pdf->GetName())) { tool->queueExport(*pdf); channelConstraintNames.push_back({pdf->GetName()}); } @@ -1746,31 +1607,10 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname return true; } -class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { +// Writes a RooRealSumPdf, or the product of one with its constraint terms, as a histfactory_dist. +class HistFactoryStreamer : public RooFit::JSONIO::Exporter { public: bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooProdPdf *prodpdf) const - { - std::vector constraints; - RooRealSumPdf *sumpdf = nullptr; - for (auto *pdf : static_range_cast(prodpdf->pdfList())) { - auto thispdf = dynamic_cast(pdf); - if (thispdf) { - if (!sumpdf) { - sumpdf = thispdf; - } else { - return false; - } - } else { - // The constraint terms can also be grouped in nested products. - collectConstraintTerms(pdf, constraints); - } - } - if (!sumpdf) - return false; - - return tryExportHistFactory(tool, prodpdf->GetName(), sumpdf, constraints); - } std::string const &key() const override { static const std::string keystring = "histfactory_dist"; @@ -1778,26 +1618,18 @@ class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { } bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override { - return tryExport(tool, static_cast(p)); - } -}; - -class HistFactoryStreamer_SumPdf : public RooFit::JSONIO::Exporter { -public: - bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooRealSumPdf *sumpdf) const - { + auto const *sumpdf = dynamic_cast(p); std::vector constraints; - return tryExportHistFactory(tool, sumpdf->GetName(), sumpdf, constraints); - } - std::string const &key() const override - { - static const std::string keystring = "histfactory_dist"; - return keystring; - } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override - { - return tryExport(tool, static_cast(p)); + if (auto const *prodpdf = dynamic_cast(p)) { + sumpdf = channelSum(*prodpdf); + for (auto *pdf : static_range_cast(prodpdf->pdfList())) { + if (pdf != sumpdf) { + // The constraint terms can also be grouped in nested products. + collectConstraintTerms(pdf, constraints); + } + } + } + return tryExportHistFactory(tool, p->GetName(), sumpdf, constraints); } }; @@ -1809,8 +1641,8 @@ STATIC_EXECUTE([]() { registerImporter("interpolation0d", true); registerExporter(RooStats::HistFactory::FlexibleInterpVar::Class(), true); registerExporter(PiecewiseInterpolation::Class(), true); - registerExporter(RooProdPdf::Class(), true); - registerExporter(RooRealSumPdf::Class(), true); + registerExporter(RooProdPdf::Class(), true); + registerExporter(RooRealSumPdf::Class(), true); }); } // namespace @@ -1855,33 +1687,22 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c std::string channelName = name; erasePrefix(channelName, "model_"); - std::size_t nBins = 1; - for (auto const &axis : dist.axes) { - nBins *= std::visit(overloaded{[](RegularBinning const &b) { return static_cast(b.nbins); }, - [](EdgesBinning const &b) { return b.edges.size() - 1; }}, - axis.binning); - } - RooAbsArg *mcStatObject = nullptr; RooArgSet constraints; if (!sumW.empty()) { + // The relative uncertainties. The gammas of bins without a positive yield are set constant later. std::vector errs(sumW.size()); for (std::size_t i = 0; i < sumW.size(); ++i) { - if (sumW[i] == 0.) { - errs[i] = 0.; - continue; - } - errs[i] = std::sqrt(sumW2[i]) / sumW[i]; - // avoid negative sigma. This NP will be set constant anyway later - errs[i] = std::max(errs[i], 0.); + errs[i] = sumW[i] > 0. ? std::sqrt(sumW2[i]) / sumW[i] : 0.; } - mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, - statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, - dist.statError.relThreshold, nBins, "the staterror modifier of channel '" + name + "'"); + mcStatObject = + &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, + statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, + dist.statError.relThreshold, errs.size(), "the staterror modifier of channel '" + name + "'"); } - ChannelContext ctx{tool, dist, observables, name, channelName, nBins, mcStatObject, constraints}; + ChannelContext ctx{tool, dist, observables, name, channelName, mcStatObject, constraints}; RooArgList funcs; RooArgList coefs; @@ -1894,29 +1715,19 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c if (channelConstraints) { for (auto const &ref : *channelConstraints) { - RooAbsPdf *pdf = tool.tryRequest(ref.name); - if (!pdf) { - RooJSONFactoryWSTool::error("unable to find the constraint '" + ref.name + "' of '" + name + "'"); - } - constraints.add(*pdf); + constraints.add(*tool.request(ref.name, name)); } } + // With constraints, the channel is the product of the sum of the samples and the constraints. + auto &sum = tool.wsEmplace(constraints.empty() ? name : channelName + "_model", funcs, coefs, true); + sum.SetTitle(name.c_str()); // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation // generates bin counts, so both are enabled, as HistFactory does by default. - auto enableBinned = [](RooRealSumPdf &sum) { - sum.setAttribute("BinnedLikelihood"); - sum.setAttribute("GenerateBinned"); - }; - if (constraints.empty()) { - enableBinned(tool.wsEmplace(name, funcs, coefs, true)); - } else { - std::string sumName = name + "_model"; - erasePrefix(sumName, "model_"); - auto &sum = tool.wsEmplace(sumName, funcs, coefs, true); - sum.SetTitle(name.c_str()); - enableBinned(sum); + sum.setAttribute("BinnedLikelihood"); + sum.setAttribute("GenerateBinned"); + if (!constraints.empty()) { tool.wsEmplace(name, constraints, RooFit::Conditional(sum, observables)); } } @@ -1938,11 +1749,8 @@ void RooFit::HS3::importHistFactoryNode(RooJSONFactoryWSTool &tool, JSONNode con node.writeJSON(ss); std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); - HS3::Version version; - HS3::parseVersion(tool.hs3VersionInput(), version); - HS3::Diagnostics diagnostics; - HS3::upgradeHistFactoryDist(tree->rootnode(), version, diagnostics); + HS3::upgradeHistFactoryDist(tree->rootnode(), tool.hs3Version(), diagnostics); HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); if (!HS3::hasErrors(diagnostics)) { HS3::validateHistFactoryDist(dist, name, diagnostics); diff --git a/roofit/hs3/src/JSONFactories_RooFitCore.cxx b/roofit/hs3/src/JSONFactories_RooFitCore.cxx index eb01d2e04cec8..5a3b2382ca4d4 100644 --- a/roofit/hs3/src/JSONFactories_RooFitCore.cxx +++ b/roofit/hs3/src/JSONFactories_RooFitCore.cxx @@ -17,12 +17,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -64,7 +62,6 @@ #include #include #include -#include #include #include @@ -72,6 +69,7 @@ #include #include #include +#include "HS3Builder.h" #include "JSONIOUtils.h" #include @@ -82,8 +80,6 @@ #include #include -#include -#include #include #include #include @@ -93,6 +89,8 @@ using RooFit::Detail::JSONNode; +namespace HS3 = RooFit::HS3; + /////////////////////////////////////////////////////////////////////////////////////////////////////// // individually implemented importers /////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -157,6 +155,11 @@ std::set extractArguments(std::string expr) return arguments; } +bool isIdentifierChar(char c) +{ + return std::isalnum(static_cast(c)) || c == '_'; +} + void replaceIdentifier(TString &expr, std::string_view identifier, std::string_view replacement) { std::string in(expr.Data()); @@ -165,11 +168,9 @@ void replaceIdentifier(TString &expr, std::string_view identifier, std::string_v for (std::size_t pos = 0; pos < in.size();) { const bool matches = in.compare(pos, identifier.size(), identifier) == 0; - const bool beforeIdentifier = - pos > 0 && (std::isalnum(static_cast(in[pos - 1])) || in[pos - 1] == '_'); + const bool beforeIdentifier = pos > 0 && isIdentifierChar(in[pos - 1]); const std::size_t end = pos + identifier.size(); - const bool afterIdentifier = - end < in.size() && (std::isalnum(static_cast(in[end])) || in[end] == '_'); + const bool afterIdentifier = end < in.size() && isIdentifierChar(in[end]); if (matches && !beforeIdentifier && !afterIdentifier) { out.append(replacement); pos = end; @@ -182,11 +183,6 @@ void replaceIdentifier(TString &expr, std::string_view identifier, std::string_v expr = out.c_str(); } -bool isIdentifierChar(char c) -{ - return std::isalnum(static_cast(c)) || c == '_'; -} - std::string trimmed(std::string const &str) { const auto begin = str.find_first_not_of(" \t\n"); @@ -275,117 +271,39 @@ void translateImportedExpression(TString &expr) { replaceIdentifier(expr, "PI", "TMath::Pi()"); replaceIdentifier(expr, "EULER", "TMath::E()"); - if (std::string(expr.Data()).find("ifelse") != std::string::npos) { - expr = ifElseToTernary(expr.Data()).c_str(); - } -} - -int readPositiveInteger(const JSONNode &node, const std::string &context) -{ - // Read through val_double() so an integer encoded as a JSON float (e.g. 1e6, - // whose textual form is "1e+06") is accepted like elsewhere in HS3, while - // fractional, non-finite, out-of-range or non-numeric values are rejected. - const double value = node.is_number() ? node.val_double() : std::numeric_limits::quiet_NaN(); - if (!std::isfinite(value) || value < 1.0 || value != std::floor(value) || - value > static_cast(std::numeric_limits::max())) { - RooJSONFactoryWSTool::error("\"nbins\" in " + context + " must be a positive integer"); - } - return static_cast(value); -} - -std::unique_ptr -readFormulaAxisBinning(const JSONNode &axis, const std::string &axisName, const std::string &formulaName) -{ - const std::string context = "axis '" + axisName + "' of generic formula '" + formulaName + "'"; - const bool hasEdges = axis.has_child("edges"); - const bool hasMin = axis.has_child("min"); - const bool hasMax = axis.has_child("max"); - const bool hasNBins = axis.has_child("nbins"); - - if (hasEdges && hasNBins) { - RooJSONFactoryWSTool::error(context + " must use either \"edges\" or \"nbins\""); - } - - if (hasEdges) { - const JSONNode &edgesNode = axis["edges"]; - if (!edgesNode.is_seq()) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must be a sequence"); - } - - std::vector edges; - edges.reserve(edgesNode.num_children()); - for (const JSONNode &edgeNode : edgesNode.children()) { - if (!edgeNode.is_number()) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain only finite values"); - } - const double edge = edgeNode.val_double(); - if (!std::isfinite(edge)) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain only finite values"); - } - if (!edges.empty() && edge <= edges.back()) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must be strictly increasing"); - } - edges.push_back(edge); - } - if (edges.size() < 2) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain at least two values"); - } - // The HS3 standard requires "min" and "max" also for irregular binnings, which older versions of RooFit didn't - // write. - if ((hasMin && (!axis["min"].is_number() || axis["min"].val_double() != edges.front())) || - (hasMax && (!axis["max"].is_number() || axis["max"].val_double() != edges.back()))) { - RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must agree with the \"edges\""); - } - return std::make_unique(static_cast(edges.size() - 1), edges.data()); - } - - if (!hasMin || !hasMax || !hasNBins) { - RooJSONFactoryWSTool::error(context + " must define \"min\", \"max\", and \"nbins\""); - } - - if (!axis["min"].is_number() || !axis["max"].is_number()) { - RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must be finite and increasing"); - } - const double min = axis["min"].val_double(); - const double max = axis["max"].val_double(); - if (!std::isfinite(min) || !std::isfinite(max) || max <= min) { - RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must be finite and increasing"); - } - return std::make_unique(min, max, readPositiveInteger(axis["nbins"], context)); + expr = ifElseToTernary(expr.Data()).c_str(); } +// Give the formula the binnings of its "axes", which are read like the axes of binned data. template void importFormulaBinnings(RooArg_t &arg, const JSONNode &node) { if (!node.has_child("axes")) { return; } - - const JSONNode &axes = node["axes"]; - if (!axes.is_seq()) { - RooJSONFactoryWSTool::error("\"axes\" in generic formula '" + std::string(arg.GetName()) + - "' must be a sequence"); - } + const std::string context = "generic formula '" + std::string(arg.GetName()) + "'"; + HS3::Diagnostics diagnostics; + const std::vector axes = HS3::parseBinnedAxes(node["axes"], diagnostics); + HS3::report(diagnostics, std::string(arg.GetName()) + ".axes"); std::set axisNames; - for (const JSONNode &axis : axes.children()) { - if (!axis.is_map() || !axis.has_child("name")) { - RooJSONFactoryWSTool::error("each axis in generic formula '" + std::string(arg.GetName()) + - "' must be a map with a \"name\""); + for (HS3::BinnedAxis const &axis : axes) { + if (!axisNames.insert(axis.name).second) { + RooJSONFactoryWSTool::error("duplicate axis '" + axis.name + "' in " + context); } - const std::string axisName = axis["name"].val(); - if (!axisNames.insert(axisName).second) { - RooJSONFactoryWSTool::error("duplicate axis '" + axisName + "' in generic formula '" + arg.GetName() + "'"); + if (!dynamic_cast(arg.getParameter(axis.name.c_str()))) { + RooJSONFactoryWSTool::error("axis '" + axis.name + "' is not a real-valued formula variable of " + context); } - - auto *observable = dynamic_cast(arg.getParameter(axisName.c_str())); - if (!observable) { - RooJSONFactoryWSTool::error( - "axis '" + axisName + "' is not a real-valued formula variable of generic formula '" + arg.GetName() + "'"); + auto const *regular = std::get_if(&axis.binning); + if (regular && (regular->nbins < 1 || !(axis.min < axis.max))) { + RooJSONFactoryWSTool::error("axis '" + axis.name + "' of " + context + + " needs a positive \"nbins\" and \"min\" smaller than \"max\""); } - - std::unique_ptr binning = readFormulaAxisBinning(axis, axisName, arg.GetName()); - arg.setBinning(*observable, *binning, /*checkFlatness=*/false); + } + // The formula finds its variable by name, so the variables that are created for the axes can stand in for them. + RooArgSet vars = HS3::createObservables(axes); + for (auto *var : static_range_cast(vars)) { + arg.setBinning(*var, var->getBinning(), /*checkFlatness=*/false); } } @@ -559,23 +477,13 @@ bool importArrayDist(RooJSONFactoryWSTool *tool, const JSONNode &p) return true; } -// One field of a component of an array distribution: a name or a number. -using ArrayFieldValue = std::variant; - +// A component of an array distribution, with the names or numbers of its fields. struct ArrayComponent { std::string type; std::string x; - std::vector> fields; + std::vector> fields; }; -ArrayFieldValue nameOrNumber(RooAbsReal const &arg) -{ - if (isLiteralConstVar(arg)) { - return static_cast(arg).getVal(); - } - return std::string{arg.GetName()}; -} - // The type, observable and fields of `pdf` as a component of an array distribution, with the fields of the standalone // exporters, or std::nullopt if `pdf` cannot be a component. A Gaussian whose mean is the recorded `observable` is // written with x and mean swapped. @@ -586,7 +494,7 @@ std::optional arrayComponentOf(RooAbsPdf const &pdf, std::string if (auto const *poisson = dynamic_cast(&pdf)) { out.type = poisson->getNoRounding() ? "poisson_continuous_dist" : "poisson_dist"; x = &poisson->getX(); - out.fields = {{"mean", nameOrNumber(poisson->getMean())}}; + out.fields = {{"mean", HS3::toNameOrNumber(poisson->getMean())}}; } else if (auto const *gauss = dynamic_cast(&pdf)) { out.type = "gaussian_dist"; x = &gauss->getX(); @@ -596,14 +504,15 @@ std::optional arrayComponentOf(RooAbsPdf const &pdf, std::string if (observable && *observable == mean->GetName() && !isLiteralConstVar(*mean)) { std::swap(x, mean); } - out.fields = {{"mean", nameOrNumber(*mean)}, {"sigma", nameOrNumber(gauss->getSigma())}}; + out.fields = {{"mean", HS3::toNameOrNumber(*mean)}, {"sigma", HS3::toNameOrNumber(gauss->getSigma())}}; } else if (auto const *lognormal = dynamic_cast(&pdf)) { if (!lognormal->useStandardParametrization()) { return std::nullopt; } out.type = "lognormal_dist"; x = &lognormal->getX(); - out.fields = {{"mu", nameOrNumber(lognormal->getMedian())}, {"sigma", nameOrNumber(lognormal->getShapeK())}}; + out.fields = {{"mu", HS3::toNameOrNumber(lognormal->getMedian())}, + {"sigma", HS3::toNameOrNumber(lognormal->getShapeK())}}; } else { return std::nullopt; } @@ -799,7 +708,7 @@ bool importPoisson(RooJSONFactoryWSTool *tool, const JSONNode &p) bool noRounding = Continuous; if (p.has_child("integer")) { std::string const type = Continuous ? "poisson_continuous_dist" : "poisson_dist"; - if (Continuous || !isHS3VersionBefore(tool->hs3VersionInput(), 0, 3)) { + if (Continuous || !(tool->hs3Version() < HS3::Version{0, 3})) { RooJSONFactoryWSTool::error( type + " '" + name + "' has the key \"integer\", which is not part of HS3. The discrete Poisson distribution is " @@ -857,14 +766,10 @@ bool importRealIntegral(RooJSONFactoryWSTool *tool, const JSONNode &p) normSet.add(tool->requestArgSet(p, "normalization")); normSetPtr = &normSet; } - std::string domain; - bool hasDomain = p.has_child("domain"); - if (hasDomain) { - domain = p["domain"].val(); - } + JSONNode const *domain = p.find("domain"); // todo: at some point, take care of integrator configurations tool->wsEmplace(name, *func, vars, normSetPtr, static_cast(nullptr), - hasDomain ? domain.c_str() : nullptr); + domain ? domain->val().c_str() : nullptr); return true; } @@ -875,13 +780,11 @@ bool importDerivative(RooJSONFactoryWSTool *tool, const JSONNode &p) RooRealVar *x = tool->requestArg(p, "x"); Int_t order = p["order"].val_int(); double eps = p["eps"].val_double(); + RooArgSet normSet; if (p.has_child("normalization")) { - RooArgSet normSet; normSet.add(tool->requestArgSet(p, "normalization")); - tool->wsEmplace(name, *func, *x, normSet, order, eps); - return true; } - tool->wsEmplace(name, *func, *x, order, eps); + tool->wsEmplace(name, *func, *x, normSet, order, eps); return true; } @@ -907,12 +810,8 @@ bool importExtendPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) // HS3 calls the fields "dist" and "rate". RooFit used to write "pdf" and "norm", which are still accepted. RooAbsPdf *pdf = tool->requestArg(p, p.has_child("pdf") && !p.has_child("dist") ? "pdf" : "dist"); RooAbsReal *norm = tool->requestArg(p, p.has_child("norm") && !p.has_child("rate") ? "norm" : "rate"); - if (p.has_child("range")) { - std::string rangeName = p["range"].val(); - tool->wsEmplace(name, *pdf, *norm, rangeName.c_str()); - return true; - } - tool->wsEmplace(name, *pdf, *norm); + JSONNode const *range = p.find("range"); + tool->wsEmplace(name, *pdf, *norm, range ? range->val().c_str() : nullptr); return true; } @@ -984,54 +883,37 @@ bool importExponential(RooJSONFactoryWSTool *tool, const JSONNode &p) bool importMultiVarGaussian(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); - bool has_cov = p.has_child("covariances"); - bool has_corr = p.has_child("correlations") && p.has_child("standard_deviations"); - if (!has_cov && !has_corr) { + const bool hasCov = p.has_child("covariances"); + const bool hasCorr = p.has_child("correlations") && p.has_child("standard_deviations"); + if (!hasCov && !hasCorr) { RooJSONFactoryWSTool::error("no covariances or correlations+standard_deviations given in '" + name + "'"); } - - TMatrixDSym covmat; - + // The covariances are given directly, or as correlations that are scaled with the standard deviations. + const std::string key = hasCov ? "covariances" : "correlations"; + std::vector sigmas; + if (hasCov) { + sigmas.assign(p[key].num_children(), 1.0); + } else { + sigmas << p["standard_deviations"]; + } + const std::size_t n = sigmas.size(); // The matrix must be square, otherwise the entries would be written out of bounds. - auto checkSquare = [&](const char *key, std::size_t n) { - if (p[key].num_children() != n) { - RooJSONFactoryWSTool::error("\"" + std::string(key) + "\" in '" + name + "' must have " + std::to_string(n) + - " rows"); - } - for (const auto &row : p[key].children()) { - if (!row.is_seq() || row.num_children() != n) { - RooJSONFactoryWSTool::error("\"" + std::string(key) + "\" in '" + name + "' must be a square matrix"); - } - } - }; - - if (has_cov) { - int n = p["covariances"].num_children(); - checkSquare("covariances", n); - int i = 0; - covmat.ResizeTo(n, n); - for (const auto &row : p["covariances"].children()) { - int j = 0; - for (const auto &val : row.children()) { - covmat(i, j) = val.val_double(); - ++j; - } - ++i; + if (p[key].num_children() != n) { + RooJSONFactoryWSTool::error("\"" + key + "\" in '" + name + "' must have " + std::to_string(n) + " rows"); + } + TMatrixDSym covmat; + covmat.ResizeTo(n, n); + int i = 0; + for (const auto &row : p[key].children()) { + if (!row.is_seq() || row.num_children() != n) { + RooJSONFactoryWSTool::error("\"" + key + "\" in '" + name + "' must be a square matrix"); } - } else { - std::vector variances; - variances << p["standard_deviations"]; - checkSquare("correlations", variances.size()); - covmat.ResizeTo(variances.size(), variances.size()); - int i = 0; - for (const auto &row : p["correlations"].children()) { - int j = 0; - for (const auto &val : row.children()) { - covmat(i, j) = val.val_double() * variances[i] * variances[j]; - ++j; - } - ++i; + int j = 0; + for (const auto &val : row.children()) { + covmat(i, j) = val.val_double() * sigmas[i] * sigmas[j]; + ++j; } + ++i; } tool->wsEmplace(name, tool->requestArgList(p, "x"), tool->requestArgList(p, "mean"), covmat); @@ -1064,13 +946,13 @@ bool importParamHistFunc(RooJSONFactoryWSTool *tool, const JSONNode &p) << ". Using default binning (uniform; nbins=100). If needed, export the Workspace to JSON with a newer " << "Root version that supports custom ParamHistFunc binnings(>=6.38.00)." << std::endl; RooJSONFactoryWSTool::warning(ss.str()); - checkParamHistFuncOrderIsDefined(tool->hs3VersionInput(), varList, "the ParamHistFunc '" + name + "'"); + checkParamHistFuncOrderIsDefined(tool->hs3Version(), varList, "the ParamHistFunc '" + name + "'"); tool->wsEmplace(name, varList, toParamHistFuncOrder(tool->requestArgList(p, "parameters"), varList)); return true; } RooArgList binning = readBinning(p, varList); - checkParamHistFuncOrderIsDefined(tool->hs3VersionInput(), binning, "the ParamHistFunc '" + name + "'"); + checkParamHistFuncOrderIsDefined(tool->hs3Version(), binning, "the ParamHistFunc '" + name + "'"); // The parameters are given in the order of HS3 (last axis fastest); ParamHistFunc expects TH1 order. tool->wsEmplace(name, binning, toParamHistFuncOrder(tool->requestArgList(p, "parameters"), binning)); @@ -1080,32 +962,20 @@ bool importParamHistFunc(RooJSONFactoryWSTool *tool, const JSONNode &p) bool importSpline(RooJSONFactoryWSTool *tool, const JSONNode &p) { const std::string name(RooJSONFactoryWSTool::name(p)); - - // Mandatory fields - if (!p.has_child("x")) { - RooJSONFactoryWSTool::error("no x given in '" + name + "'"); - } + RooAbsReal *x = tool->requestArg(p, "x"); if (!p.has_child("x0") || !p.has_child("y0")) { RooJSONFactoryWSTool::error("no x0/y0 given in '" + name + "'"); } - RooAbsReal *x = tool->requestArg(p, "x"); - - // Optional fields (defaults follow RooSpline ctor defaults) - std::string algo = p.has_child("interpolation") ? p["interpolation"].val() : "poly3"; - int order = 0; - if (algo == "poly3") { - order = 3; - } else if (algo == "poly5") { - order = 5; - } else { + // The optional fields have the defaults of RooSpline. + const std::string algo = p.has_child("interpolation") ? p["interpolation"].val() : "poly3"; + if (algo != "poly3" && algo != "poly5") { RooJSONFactoryWSTool::error("unsupported algo '" + algo + "' for RooSpline in '" + name + "': allowed are 'poly3' and 'poly5'"); } - const bool logx = p.has_child("logx") ? p["logx"].val_bool() : false; - const bool logy = p.has_child("logy") ? p["logy"].val_bool() : false; + const bool logx = p.has_child("logx") && p["logx"].val_bool(); + const bool logy = p.has_child("logy") && p["logy"].val_bool(); - // Read knots std::vector x0; std::vector y0; x0 << p["x0"]; @@ -1119,10 +989,7 @@ bool importSpline(RooJSONFactoryWSTool *tool, const JSONNode &p) RooJSONFactoryWSTool::error("need at least 2 knots in '" + name + "'"); } - // Construct RooSpline(name,title, x, x0, y0, order, logx, logy) - tool->wsEmplace<::RooSpline>(name.c_str(), *x, std::span(x0.data(), x0.size()), - std::span(y0.data(), y0.size()), order, logx, logy); - + tool->wsEmplace<::RooSpline>(name.c_str(), *x, x0, y0, algo == "poly5" ? 5 : 3, logx, logy); return true; } @@ -1237,8 +1104,8 @@ bool importUniform(RooJSONFactoryWSTool *tool, const JSONNode &p) return true; } -// Importer that is registered with top priority for a deprecated type key. It warns once, and then lets the importers -// that are registered for the deprecated key do the actual work. +// Importer that is registered first for a deprecated type key. It warns once, and then lets the importers that are +// registered after it for the deprecated key do the actual work. class DeprecatedKeyImporter : public RooFit::JSONIO::Importer { public: DeprecatedKeyImporter(std::string key, std::string replacement) @@ -1335,33 +1202,28 @@ void importDoubleSided(RooJSONFactoryWSTool *tool, const JSONNode &p, std::strin // "crystalball_dist" is the single-sided Crystal Ball (RooCBShape). Before HS3 0.3, it was also the double-sided // Crystal Ball, recognizable by the "_L"/"_R" fields. Such documents are still read, with a deprecation warning. -class CrystalBallImporter : public RooFit::JSONIO::Importer { -public: - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override - { - if (!CrystalBall::hasDoubleSidedKeys(p)) { - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *m = tool->requestArg(p, "m"); - RooAbsReal *m0 = tool->requestArg(p, "m0"); - RooAbsReal *sigma = tool->requestArg(p, "sigma"); - RooAbsReal *alpha = tool->requestArg(p, "alpha"); - RooAbsReal *n = tool->requestArg(p, "n"); - tool->wsEmplace(name, *m, *m0, *sigma, *alpha, *n); - return true; - } - if (!_warned) { - RooJSONFactoryWSTool::warning("RooFitHS3: \"crystalball_dist\" with the fields \"alpha_L\", \"n_L\", " - "\"alpha_R\", \"n_R\", \"sigma_L\" or \"sigma_R\" is deprecated, please use " - "\"crystalball_doublesided_dist\" for the double-sided Crystal Ball."); - _warned = true; - } - CrystalBall::importDoubleSided(tool, p, "crystalball_dist", /*symmetricShorthand=*/true); +bool importCrystalBall(RooJSONFactoryWSTool *tool, const JSONNode &p) +{ + if (!CrystalBall::hasDoubleSidedKeys(p)) { + std::string name(RooJSONFactoryWSTool::name(p)); + RooAbsReal *m = tool->requestArg(p, "m"); + RooAbsReal *m0 = tool->requestArg(p, "m0"); + RooAbsReal *sigma = tool->requestArg(p, "sigma"); + RooAbsReal *alpha = tool->requestArg(p, "alpha"); + RooAbsReal *n = tool->requestArg(p, "n"); + tool->wsEmplace(name, *m, *m0, *sigma, *alpha, *n); return true; } - -private: - mutable bool _warned = false; -}; + static bool warned = false; + if (!warned) { + RooJSONFactoryWSTool::warning("RooFitHS3: \"crystalball_dist\" with the fields \"alpha_L\", \"n_L\", " + "\"alpha_R\", \"n_R\", \"sigma_L\" or \"sigma_R\" is deprecated, please use " + "\"crystalball_doublesided_dist\" for the double-sided Crystal Ball."); + warned = true; + } + CrystalBall::importDoubleSided(tool, p, "crystalball_dist", /*symmetricShorthand=*/true); + return true; +} bool importCrystalBallDoubleSided(RooJSONFactoryWSTool *tool, const JSONNode &p) { @@ -1376,67 +1238,55 @@ bool importCrystalBallDoubleSided(RooJSONFactoryWSTool *tool, const JSONNode &p) // A RooCrystalBall with one tail is the single-sided "crystalball_dist" (read back as RooCBShape, which follows the // same convention for the sign of alpha). With two tails, it is a "crystalball_doublesided_dist", whose tails are // always written as "alpha_L"/"n_L"/"alpha_R"/"n_R": there is no shorthand for symmetric tails. -bool exportCrystalBall(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportCrystalBall(RooJSONFactoryWSTool *, RooCrystalBall const &pdf, JSONNode &elem) { - auto const *pdf = static_cast(func); - writeReference(elem["m"], pdf->x()); - writeReference(elem["m0"], pdf->x0()); - if (!pdf->alphaR() || !pdf->nR()) { + writeReference(elem["m"], pdf.x()); + writeReference(elem["m0"], pdf.x0()); + if (!pdf.alphaR() || !pdf.nR()) { elem["type"] << "crystalball_dist"; - writeReference(elem["sigma"], pdf->sigmaL()); - writeReference(elem["alpha"], pdf->alphaL()); - writeReference(elem["n"], pdf->nL()); - return true; + writeReference(elem["sigma"], pdf.sigmaL()); + writeReference(elem["alpha"], pdf.alphaL()); + writeReference(elem["n"], pdf.nL()); + return; } - elem["type"] << key; - if (&pdf->sigmaL() == &pdf->sigmaR()) { - writeReference(elem["sigma"], pdf->sigmaL()); + if (&pdf.sigmaL() == &pdf.sigmaR()) { + writeReference(elem["sigma"], pdf.sigmaL()); } else { - writeReference(elem["sigma_L"], pdf->sigmaL()); - writeReference(elem["sigma_R"], pdf->sigmaR()); + writeReference(elem["sigma_L"], pdf.sigmaL()); + writeReference(elem["sigma_R"], pdf.sigmaR()); } - writeReference(elem["alpha_L"], pdf->alphaL()); - writeReference(elem["n_L"], pdf->nL()); - writeReference(elem["alpha_R"], *pdf->alphaR()); - writeReference(elem["n_R"], *pdf->nR()); - return true; + writeReference(elem["alpha_L"], pdf.alphaL()); + writeReference(elem["n_L"], pdf.nL()); + writeReference(elem["alpha_R"], *pdf.alphaR()); + writeReference(elem["n_R"], *pdf.nR()); } template -bool exportAddPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportAddPdf(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { - const RooArg_t *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["summands"], pdf->pdfList()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - elem["extended"] << (pdf->extendMode() != RooArg_t::CanNotBeExtended); - return true; + RooJSONFactoryWSTool::fillSeq(elem["summands"], pdf.pdfList()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); + elem["extended"] << (pdf.extendMode() != RooArg_t::CanNotBeExtended); } template -bool exportRealSum(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportRealSum(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { - auto const *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["samples"], pdf->funcList()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); + RooJSONFactoryWSTool::fillSeq(elem["samples"], pdf.funcList()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); if constexpr (std::is_same_v) { - elem["extended"] << (pdf->extendMode() != RooAbsPdf::CanNotBeExtended); + elem["extended"] << (pdf.extendMode() != RooAbsPdf::CanNotBeExtended); } - return true; } template -bool exportHist(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportHist(RooJSONFactoryWSTool *tool, RooArg_t const &hf, JSONNode &elem) { - const RooArg_t *hf = static_cast(func); - elem["type"] << key; - RooDataHist const &dh = hf->dataHist(); + RooDataHist const &dh = hf.dataHist(); // The axes are named after the observables of the function, which are not necessarily the variables of the // histogram. - auto vars = histogramVariablesNamedAfterObservables(dh, hf->getHistObsList(), hf->variables()); + auto vars = histogramVariablesNamedAfterObservables(dh, hf.getHistObsList(), hf.variables()); tool->exportHisto(*vars, dh.numEntries(), dh.weightArray(), elem["data"].set_map()); - return true; } template @@ -1452,22 +1302,20 @@ bool importHist(RooJSONFactoryWSTool *tool, const JSONNode &p) return true; } -bool exportBinSamplingPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportBinSamplingPdf(RooJSONFactoryWSTool *, RooBinSamplingPdf const &pdf, JSONNode &elem) { - const RooBinSamplingPdf *pdf = static_cast(func); - elem["type"] << key; - elem["pdf"] << pdf->pdf().GetName(); - elem["observable"] << pdf->observable().GetName(); - elem["epsilon"] << pdf->epsilon(); - return true; + elem["pdf"] << pdf.pdf().GetName(); + elem["observable"] << pdf.observable().GetName(); + elem["epsilon"] << pdf.epsilon(); } -bool exportBinWidthFunction(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &) +void exportBinWidthFunction(RooJSONFactoryWSTool *, RooBinWidthFunction const &pdf, JSONNode &elem) { - const RooBinWidthFunction *pdf = static_cast(func); - elem["type"] << (pdf->divideByBinWidth() ? "inverse_binvolume" : "binvolume"); - elem["histogram"] << pdf->histFunc().GetName(); - return true; + // It is registered for "binvolume". + if (pdf.divideByBinWidth()) { + elem["type"] << "inverse_binvolume"; + } + elem["histogram"] << pdf.histFunc().GetName(); } void cleanExpression(TString &expr) @@ -1620,11 +1468,9 @@ bool ternariesToIfElse(std::string const &in, std::string &out) } template -bool exportFormulaArg(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportFormulaArg(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { - const RooArg_t *pdf = static_cast(func); - elem["type"] << key; - TString expression(pdf->expression()); + TString expression(pdf.expression()); cleanExpression(expression); // If the tokens follow the "x[#]" convention, the square braces enclosing each number // ensures that there is a unique mapping between the token and parameter name @@ -1632,8 +1478,8 @@ bool exportFormulaArg(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &e // So there may be tokens with numbers whose lower place value forms a subset string of ones with a higher place // value, e.g. "@1" is a subset of "@10". So the names of these parameters must be applied descending from the // highest place value in order to ensure each parameter name is uniquely applied to its token. - for (size_t idx = pdf->nParameters(); idx--;) { - const RooAbsArg *par = pdf->getParameter(idx); + for (size_t idx = pdf.nParameters(); idx--;) { + const RooAbsArg *par = pdf.getParameter(idx); expression.ReplaceAll(("x[" + std::to_string(idx) + "]").c_str(), par->GetName()); expression.ReplaceAll(("@" + std::to_string(idx)).c_str(), par->GetName()); } @@ -1644,453 +1490,328 @@ bool exportFormulaArg(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &e } else { RooJSONFactoryWSTool::warning(std::string("RooFitHS3: could not translate the ternary operators in the " "expression of '") + - func->GetName() + "' to ifelse(), exporting the expression unchanged."); + pdf.GetName() + "' to ifelse(), exporting the expression unchanged."); } } elem["expression"] << expression.Data(); - for (const RooAbsArg *dependent : pdf->dependents()) { + for (const RooAbsArg *dependent : pdf.dependents()) { auto const *observable = dynamic_cast(dependent); if (!observable) { continue; } - const RooAbsBinning *binning = pdf->getBinning(*observable); + const RooAbsBinning *binning = pdf.getBinning(*observable); if (!binning) { continue; } - auto &axes = elem["axes"]; - if (!axes.is_seq()) { - axes.set_seq(); - } - auto &axis = axes.append_child().set_map(); - axis["name"] << observable->GetName(); - writeAxisBinning(axis, *binning); + HS3::writeAxis(elem["axes"].set_seq().append_child(), HS3::toBinnedAxis(observable->GetName(), *binning)); } - return true; } // Write the "x" reference and the coefficient list for polynomial-like // pdfs/funcs, including the implicit defaults below "lowestOrder" so that the // output is self-documenting. template -bool exportPolynomial(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportPolynomial(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { - auto const *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->x().GetName(); + elem["x"] << pdf.x().GetName(); auto &coefs = elem["coefficients"].set_seq(); - for (int i = 0; i < pdf->lowestOrder(); ++i) { + for (int i = 0; i < pdf.lowestOrder(); ++i) { coefs.append_child() << (i == 0 ? 1.0 : 0.0); } - for (const auto &coef : pdf->coefList()) { + for (const auto &coef : pdf.coefList()) { coefs.append_child() << coef->GetName(); } - return true; } -bool exportPoisson(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportPoisson(RooJSONFactoryWSTool *, RooPoisson const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << (pdf->getNoRounding() ? "poisson_continuous_dist" : key); - elem["x"] << pdf->getX().GetName(); - elem["mean"] << pdf->getMean().GetName(); - return true; + // It is registered for "poisson_dist", the discrete Poisson distribution. + if (pdf.getNoRounding()) { + elem["type"] << "poisson_continuous_dist"; + } + elem["x"] << pdf.getX().GetName(); + elem["mean"] << pdf.getMean().GetName(); } // The RooDecay servers are the internal resModel-times-basis convolutions, // which are implementation details that should not leak into the JSON. We // only export the actual dependents (t, tau and the original resolution // model) explicitly. -bool exportDecay(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportDecay(RooJSONFactoryWSTool *tool, RooDecay const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["t"] << pdf->getT().GetName(); - elem["tau"] << pdf->getTau().GetName(); - elem["resolutionModel"] << pdf->getModel().GetName(); - elem["decayType"] << pdf->getDecayType(); - - tool->queueExport(pdf->getT()); - tool->queueExport(pdf->getTau()); - tool->queueExport(pdf->getModel()); + elem["t"] << pdf.getT().GetName(); + elem["tau"] << pdf.getTau().GetName(); + elem["resolutionModel"] << pdf.getModel().GetName(); + elem["decayType"] << pdf.getDecayType(); - return true; + tool->queueExport(pdf.getT()); + tool->queueExport(pdf.getTau()); + tool->queueExport(pdf.getModel()); } -bool exportTruthModel(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportTruthModel(RooJSONFactoryWSTool *, RooTruthModel const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->convVar().GetName(); - - return true; + elem["x"] << pdf.convVar().GetName(); } -bool exportGaussModel(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportGaussModel(RooJSONFactoryWSTool *, RooGaussModel const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->convVar().GetName(); - elem["mean"] << pdf->getMean().GetName(); - elem["sigma"] << pdf->getSigma().GetName(); - return true; + elem["x"] << pdf.convVar().GetName(); + elem["mean"] << pdf.getMean().GetName(); + elem["sigma"] << pdf.getSigma().GetName(); } // A Gaussian constraint whose recorded observable (see RooJSONFactoryWSTool::setConstraintObservable()) is its // mean is written with x and mean swapped, so that the observable is x. -bool exportGaussian(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto const *gauss = static_cast(func); - std::string const *observable = tool->constraintObservable(gauss->GetName()); - const bool swap = observable && *observable == gauss->getMean().GetName() && !isLiteralConstVar(gauss->getMean()); - elem["type"] << key; - writeReference(elem["x"], swap ? gauss->getMean() : gauss->getX()); - writeReference(elem["mean"], swap ? gauss->getX() : gauss->getMean()); - writeReference(elem["sigma"], gauss->getSigma()); - return true; +void exportGaussian(RooJSONFactoryWSTool *tool, RooGaussian const &gauss, JSONNode &elem) +{ + std::string const *observable = tool->constraintObservable(gauss.GetName()); + const bool swap = observable && *observable == gauss.getMean().GetName() && !isLiteralConstVar(gauss.getMean()); + writeReference(elem["x"], swap ? gauss.getMean() : gauss.getX()); + writeReference(elem["mean"], swap ? gauss.getX() : gauss.getMean()); + writeReference(elem["sigma"], gauss.getSigma()); } -bool exportLogNormal(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportLogNormal(RooJSONFactoryWSTool *tool, RooLognormal const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - - elem["type"] << key; - elem["x"] << pdf->getX().GetName(); + elem["x"] << pdf.getX().GetName(); - auto &m0 = pdf->getMedian(); - auto &k = pdf->getShapeK(); + auto &m0 = pdf.getMedian(); + auto &k = pdf.getShapeK(); - if (pdf->useStandardParametrization()) { + if (pdf.useStandardParametrization()) { elem["mu"] << m0.GetName(); elem["sigma"] << k.GetName(); } else { elem["mu"] << tool->exportTransformed(&m0, "_lognormal_log", "log(%s)"); elem["sigma"] << tool->exportTransformed(&k, "_lognormal_log", "log(%s)"); } - - return true; } -bool exportExponential(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportExponential(RooJSONFactoryWSTool *tool, RooExponential const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->variable().GetName(); - auto &c = pdf->coefficient(); - if (pdf->negateCoefficient()) { + elem["x"] << pdf.variable().GetName(); + auto &c = pdf.coefficient(); + if (pdf.negateCoefficient()) { elem["c"] << c.GetName(); } else { elem["c"] << tool->exportTransformed(&c, "_exponential_inverted", "-%s"); } - - return true; } -bool exportMultiVarGaussian(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportMultiVarGaussian(RooJSONFactoryWSTool *, RooMultiVarGaussian const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["x"], pdf->xVec()); - RooJSONFactoryWSTool::fillSeq(elem["mean"], pdf->muVec()); - elem["covariances"].fill_mat(pdf->covarianceMatrix()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["x"], pdf.xVec()); + RooJSONFactoryWSTool::fillSeq(elem["mean"], pdf.muVec()); + elem["covariances"].fill_mat(pdf.covarianceMatrix()); } -bool exportTFnBinding(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportTFnBinding(RooJSONFactoryWSTool *, RooTFnBinding const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - TString formula(pdf->function().GetExpFormula()); - formula.ReplaceAll("x", pdf->observables()[0].GetName()); - formula.ReplaceAll("y", pdf->observables()[1].GetName()); - formula.ReplaceAll("z", pdf->observables()[2].GetName()); - for (size_t i = 0; i < pdf->parameters().size(); ++i) { + TString formula(pdf.function().GetExpFormula()); + formula.ReplaceAll("x", pdf.observables()[0].GetName()); + formula.ReplaceAll("y", pdf.observables()[1].GetName()); + formula.ReplaceAll("z", pdf.observables()[2].GetName()); + for (size_t i = 0; i < pdf.parameters().size(); ++i) { TString pname(TString::Format("[%d]", (int)i)); - formula.ReplaceAll(pname, pdf->parameters()[i].GetName()); + formula.ReplaceAll(pname, pdf.parameters()[i].GetName()); } elem["expression"] << formula.Data(); - return true; } -bool exportDerivative(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportDerivative(RooJSONFactoryWSTool *, RooDerivative const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->getX().GetName(); - elem["function"] << pdf->getFunc().GetName(); - if (!pdf->getNset().empty()) { - RooJSONFactoryWSTool::fillSeq(elem["normalization"], pdf->getNset()); + elem["x"] << pdf.getX().GetName(); + elem["function"] << pdf.getFunc().GetName(); + if (!pdf.getNset().empty()) { + RooJSONFactoryWSTool::fillSeq(elem["normalization"], pdf.getNset()); } - elem["order"] << pdf->order(); - elem["eps"] << pdf->eps(); - return true; + elem["order"] << pdf.order(); + elem["eps"] << pdf.eps(); } -bool exportRealIntegral(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportRealIntegral(RooJSONFactoryWSTool *, RooRealIntegral const &integral, JSONNode &elem) { - auto *integral = static_cast(func); - elem["type"] << key; - std::string integrand = integral->integrand().GetName(); + std::string integrand = integral.integrand().GetName(); elem["integrand"] << integrand; - if (integral->intRange()) { - elem["domain"] << integral->intRange(); + if (integral.intRange()) { + elem["domain"] << integral.intRange(); } - RooJSONFactoryWSTool::fillSeq(elem["variables"], integral->intVars()); - if (RooArgSet const *funcNormSet = integral->funcNormSet()) { + RooJSONFactoryWSTool::fillSeq(elem["variables"], integral.intVars()); + if (RooArgSet const *funcNormSet = integral.funcNormSet()) { RooJSONFactoryWSTool::fillSeq(elem["normalization"], *funcNormSet); } - return true; } -bool exportFFTConvPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportFFTConvPdf(RooJSONFactoryWSTool *, RooFFTConvPdf const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - if (auto convFunc = pdf->getPdfConvVar()) { + if (auto convFunc = pdf.getPdfConvVar()) { elem["conv_func"] << convFunc->GetName(); } - elem["conv_var"] << pdf->getConvVar().GetName(); - elem["pdf1"] << pdf->getPdf1().GetName(); - elem["pdf2"] << pdf->getPdf2().GetName(); - elem["ipOrder"] << pdf->getInterpolationOrder(); - return true; + elem["conv_var"] << pdf.getConvVar().GetName(); + elem["pdf1"] << pdf.getPdf1().GetName(); + elem["pdf2"] << pdf.getPdf2().GetName(); + elem["ipOrder"] << pdf.getInterpolationOrder(); } -bool exportExtendPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportExtendPdf(RooJSONFactoryWSTool *, RooExtendPdf const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - if (auto rangeName = pdf->getRangeName()) { + if (auto rangeName = pdf.getRangeName()) { elem["range"] << rangeName->GetName(); } - elem["dist"] << pdf->pdf().GetName(); - elem["rate"] << pdf->getN().GetName(); - return true; + elem["dist"] << pdf.pdf().GetName(); + elem["rate"] << pdf.getN().GetName(); } -bool exportParamHistFunc(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportParamHistFunc(RooJSONFactoryWSTool *, ParamHistFunc const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["variables"], pdf->dataVars()); + RooJSONFactoryWSTool::fillSeq(elem["variables"], pdf.dataVars()); // One parameter per bin, in the order of HS3 (last axis fastest), not in ParamHistFunc's internal TH1 order. auto ¶msNode = elem["parameters"].set_seq(); - for (RooAbsReal *param : paramHistFuncParamsInBinOrder(*pdf)) { + for (RooAbsReal *param : paramHistFuncParamsInBinOrder(pdf)) { paramsNode.append_child() << param->GetName(); } auto &observablesNode = elem["axes"].set_seq(); // axes have to be ordered to get consistent bin indices - for (auto *var : static_range_cast(pdf->dataVars())) { - RooJSONFactoryWSTool::exportAxis(observablesNode.append_child().set_map(), *var); + for (auto *var : static_range_cast(pdf.dataVars())) { + HS3::writeAxis(observablesNode.append_child(), HS3::toBinnedAxis(*var)); } - return true; } -bool exportSpline(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportSpline(RooJSONFactoryWSTool *, RooSpline const &rs, JSONNode &elem) { - auto const *rs = static_cast(func); - - elem["type"] << key; + elem["x"] << rs.x().GetName(); + elem["interpolation"] << (rs.order() == 5 ? "poly5" : "poly3"); + elem["logx"] << rs.logx(); + elem["logy"] << rs.logy(); - // Independent variable - elem["x"] << rs->x().GetName(); - - // Spline configuration - // Canonical algo for RooSpline - elem["interpolation"] << (rs->order() == 5 ? "poly5" : "poly3"); - elem["logx"] << rs->logx(); - elem["logy"] << rs->logy(); - - // Serialize knots as primitive arrays - TSpline const &sp = rs->spline(); + // The knots + TSpline const &sp = rs.spline(); auto &x0 = elem["x0"].set_seq(); auto &y0 = elem["y0"].set_seq(); - - const int np = sp.GetNp(); - for (int i = 0; i < np; ++i) { + for (int i = 0; i < sp.GetNp(); ++i) { double xk = 0.0; double yk = 0.0; sp.GetKnot(i, xk, yk); x0.append_child() << xk; y0.append_child() << yk; } - - return true; } -bool exportAddition(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportAddition(RooJSONFactoryWSTool *, RooAddition const &sum, JSONNode &elem) { - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["summands"], static_cast(func)->list()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["summands"], sum.list()); } -bool exportArgus(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportArgus(RooJSONFactoryWSTool *, RooArgusBG const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["mass"], pdf->getM()); - writeReference(elem["resonance"], pdf->getM0()); - writeReference(elem["slope"], pdf->getC()); - writeReference(elem["power"], pdf->getP()); - return true; + writeReference(elem["mass"], pdf.getM()); + writeReference(elem["resonance"], pdf.getM0()); + writeReference(elem["slope"], pdf.getC()); + writeReference(elem["power"], pdf.getP()); } -bool exportBernstein(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportBernstein(RooJSONFactoryWSTool *, RooBernstein const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->x()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - return true; + writeReference(elem["x"], pdf.x()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); } -bool exportBifurGauss(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportBifurGauss(RooJSONFactoryWSTool *, RooBifurGauss const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->getX()); - writeReference(elem["mean"], pdf->getMean()); - writeReference(elem["sigmaL"], pdf->getSigmaL()); - writeReference(elem["sigmaR"], pdf->getSigmaR()); - return true; + writeReference(elem["x"], pdf.getX()); + writeReference(elem["mean"], pdf.getMean()); + writeReference(elem["sigmaL"], pdf.getSigmaL()); + writeReference(elem["sigmaR"], pdf.getSigmaR()); } -bool exportCBShape(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportCBShape(RooJSONFactoryWSTool *, RooCBShape const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["m"], pdf->getM()); - writeReference(elem["m0"], pdf->getM0()); - writeReference(elem["sigma"], pdf->getSigma()); - writeReference(elem["alpha"], pdf->getAlpha()); - writeReference(elem["n"], pdf->getN()); - return true; + writeReference(elem["m"], pdf.getM()); + writeReference(elem["m0"], pdf.getM0()); + writeReference(elem["sigma"], pdf.getSigma()); + writeReference(elem["alpha"], pdf.getAlpha()); + writeReference(elem["n"], pdf.getN()); } -bool exportChebychev(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportChebychev(RooJSONFactoryWSTool *, RooChebychev const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->x()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - return true; + writeReference(elem["x"], pdf.x()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); } -bool exportEffProd(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportEffProd(RooJSONFactoryWSTool *, RooEffProd const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["pdf"], pdf->pdf()); - writeReference(elem["eff"], pdf->eff()); - return true; + writeReference(elem["pdf"], pdf.pdf()); + writeReference(elem["eff"], pdf.eff()); } -bool exportGamma(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportGamma(RooJSONFactoryWSTool *, RooGamma const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->getX()); - writeReference(elem["gamma"], pdf->getGamma()); - writeReference(elem["beta"], pdf->getBeta()); - writeReference(elem["mu"], pdf->getMu()); - return true; + writeReference(elem["x"], pdf.getX()); + writeReference(elem["gamma"], pdf.getGamma()); + writeReference(elem["beta"], pdf.getBeta()); + writeReference(elem["mu"], pdf.getMu()); } -bool exportLandau(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportLandau(RooJSONFactoryWSTool *, RooLandau const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->getX()); - writeReference(elem["mean"], pdf->getMean()); - writeReference(elem["sigma"], pdf->getSigma()); - return true; + writeReference(elem["x"], pdf.getX()); + writeReference(elem["mean"], pdf.getMean()); + writeReference(elem["sigma"], pdf.getSigma()); } -bool exportPowerSum(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportPowerSum(RooJSONFactoryWSTool *, RooPowerSum const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->base()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - RooJSONFactoryWSTool::fillSeq(elem["exponents"], pdf->expList()); - return true; + writeReference(elem["x"], pdf.base()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); + RooJSONFactoryWSTool::fillSeq(elem["exponents"], pdf.expList()); } -bool exportProdPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportProdPdf(RooJSONFactoryWSTool *, RooProdPdf const &pdf, JSONNode &elem) { - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["factors"], static_cast(func)->pdfList()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["factors"], pdf.pdfList()); } -bool exportProduct(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportProduct(RooJSONFactoryWSTool *, RooProduct const &prod, JSONNode &elem) { - auto *prod = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["factors"], prod->realComponents()); - RooJSONFactoryWSTool::fillSeq(elem["factors"], prod->categoricalComponents()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["factors"], prod.realComponents()); + RooJSONFactoryWSTool::fillSeq(elem["factors"], prod.categoricalComponents()); } -bool exportProjectedPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportProjectedPdf(RooJSONFactoryWSTool *, RooProjectedPdf const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["input_pdf"], pdf->intPdf()); - RooJSONFactoryWSTool::fillSeq(elem["observables"], pdf->intObs()); - return true; + writeReference(elem["input_pdf"], pdf.intPdf()); + RooJSONFactoryWSTool::fillSeq(elem["observables"], pdf.intObs()); } -bool exportUniform(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportUniform(RooJSONFactoryWSTool *, RooUniform const &pdf, JSONNode &elem) { - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["x"], static_cast(func)->observables()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["x"], pdf.observables()); } -bool importWrapperPdf(RooJSONFactoryWSTool *tool, const JSONNode &node) +bool importWrapperPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) { - if (node["type"].val() != "density_function_dist") - return false; - - auto name = RooJSONFactoryWSTool::name(node); - auto *func = tool->requestArg(node, "function"); - - bool selfNormalized = false; - - auto sn = node.find("self_normalized"); + std::string name(RooJSONFactoryWSTool::name(p)); + RooAbsReal *func = tool->requestArg(p, "function"); // ROOT previously exported this key without an underscore. - if (!sn) - sn = node.find("selfnormalized"); - if (sn) - selfNormalized = sn->val_bool(); - + const char *key = p.has_child("self_normalized") ? "self_normalized" : "selfnormalized"; + const bool selfNormalized = p.has_child(key) && p[key].val_bool(); tool->wsEmplace(name, *func, selfNormalized); return true; } -bool exportWrapperPdf(RooJSONFactoryWSTool *, const RooAbsArg *arg, JSONNode &node, std::string const &key) +void exportWrapperPdf(RooJSONFactoryWSTool *, RooWrapperPdf const &pdf, JSONNode &elem) { - auto const *pdf = dynamic_cast(arg); - if (!pdf) - return false; - - node["type"] << key; - - // Proxy name in RooWrapperPdf is "_func" / "func" depending on accessor/proxy export. - // Prefer a public accessor if one exists; otherwise inspect proxies as below. - auto const *funcProxy = dynamic_cast(pdf->getProxy(0)); - if (!funcProxy || !funcProxy->absArg()) - return false; - - node["function"] << funcProxy->absArg()->GetName(); - if (pdf->selfNormalized()) - node["self_normalized"] << true; - - return true; + // RooWrapperPdf has no accessor for the wrapped function, which is its only proxy. + auto const &funcProxy = dynamic_cast(*pdf.getProxy(0)); + elem["function"] << funcProxy.absArg()->GetName(); + if (pdf.selfNormalized()) { + elem["self_normalized"] << true; + } } /////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -2098,157 +1819,137 @@ bool exportWrapperPdf(RooJSONFactoryWSTool *, const RooAbsArg *arg, JSONNode &no /////////////////////////////////////////////////////////////////////////////////////////////////////// // Adapters that wrap the plain import/export functions above into the -// RooFit::JSONIO::Importer/Exporter interface. The exporter also owns the HS3 -// type key, which is passed at registration time. -template +// RooFit::JSONIO::Importer/Exporter interface. +using ImportFunc = bool (*)(RooJSONFactoryWSTool *, const JSONNode &); + class FuncImporter : public RooFit::JSONIO::Importer { public: - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { return Func(tool, p); } + explicit FuncImporter(ImportFunc func) : _func{func} {} + bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { return _func(tool, p); } + +private: + const ImportFunc _func; }; -template +template +using ExportFunc = void (*)(RooJSONFactoryWSTool *, RooArg_t const &, JSONNode &); + +// The exporter of the class RooArg_t writes the HS3 type key that the export function is registered with. A function +// overwrites it if the type depends on the object. +template class FuncExporter : public RooFit::JSONIO::Exporter { public: - FuncExporter(std::string key, bool autoExportDependants) - : _key{std::move(key)}, _autoExportDependants{autoExportDependants} + FuncExporter(ExportFunc func, std::string key, bool autoExportDependants) + : _func{func}, _key{std::move(key)}, _autoExportDependants{autoExportDependants} { } std::string const &key() const override { return _key; } bool autoExportDependants() const override { return _autoExportDependants; } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem) const override + bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *arg, JSONNode &elem) const override { - return Func(tool, func, elem, _key); + elem["type"] << _key; + _func(tool, static_cast(*arg), elem); + return true; } private: + const ExportFunc _func; const std::string _key; const bool _autoExportDependants; }; -template -void registerImporter(const std::string &key, bool topPriority = true) +// The importers of a type key and the exporters of a class are tried in the order of their registration. +void registerImporter(const std::string &key, ImportFunc func) { - RooFit::JSONIO::registerImporter(key, std::make_unique>(), topPriority); + RooFit::JSONIO::registerImporter(key, std::make_unique(func), false); } -template -void registerExporter(TClass const *cl, std::string key, bool topPriority = true, bool autoExportDependants = true) +template +void registerExporter(std::string key, ExportFunc func, bool autoExportDependants = true) +{ + RooFit::JSONIO::registerExporter( + RooArg_t::Class(), std::make_unique>(func, std::move(key), autoExportDependants), false); +} + +// A class that is read and written with the same type key. +template +void registerType(const std::string &key, ImportFunc importFunc, ExportFunc exportFunc, + bool autoExportDependants = true) { - RooFit::JSONIO::registerExporter(cl, std::make_unique>(std::move(key), autoExportDependants), - topPriority); + registerImporter(key, importFunc); + registerExporter(key, exportFunc, autoExportDependants); +} + +// A deprecated type key is still read like its replacement, with a warning. +void registerDeprecatedImporter(const std::string &key, const std::string &replacement, ImportFunc func) +{ + RooFit::JSONIO::registerImporter(key, std::make_unique(key, replacement), false); + registerImporter(key, func); } STATIC_EXECUTE([]() { - registerImporter("density_function_dist"); - registerImporter("rate_extended_dist"); - RooFit::JSONIO::registerImporter("crystalball_dist", std::make_unique()); - registerImporter("crystalball_doublesided_dist"); - RooFit::JSONIO::registerImporter("ARGUS_dist", std::make_unique("ARGUS_dist", "argus_dist")); - RooFit::JSONIO::registerImporter("binsampling", - std::make_unique("binsampling", "binsampling_dist")); - RooFit::JSONIO::registerImporter( - "mixture_resolution_model", std::make_unique("mixture_resolution_model", "mixture_model")); - RooFit::JSONIO::registerImporter("delta_resolution_model", std::make_unique( - "delta_resolution_model", "truth_model_function")); - RooFit::JSONIO::registerImporter("gauss_resolution_model", std::make_unique( - "gauss_resolution_model", "gauss_model_function")); - registerImporter("argus_dist", false); - registerImporter("ARGUS_dist", false); - registerImporter("bernstein_poly_dist", false); - registerImporter("bifurkated_gaussian_dist", false); - registerImporter("chebychev_dist", false); - registerImporter("efficiency_product_pdf_dist", false); - registerImporter("gamma_dist", false); - registerImporter("gaussian_dist", false); - registerImporter("normal_dist", false); - registerImporter("landau_dist", false); - registerImporter("power_sum_dist", false); - registerImporter("projected_dist", false); - registerImporter("uniform_dist", false); - registerImporter("product", false); - registerImporter("product_dist", false); - registerImporter("array_dist", false); - registerImporter("sum", false); - registerImporter("mixture_dist", false); - registerImporter("mixture_model", false); - registerImporter("combined_dist", false); - registerImporter("mixture_resolution_model", false); - registerImporter("binsampling_dist", false); - registerImporter("binsampling", false); - registerImporter>("binvolume", false); - registerImporter>("inverse_binvolume", false); - registerImporter>("legacy_exp_poly_dist", false); - registerImporter("exponential_dist", false); - registerImporter>("generic", false); - registerImporter>("generic_function", false); - registerImporter>("generic_dist", false); - registerImporter>("histogram", false); - registerImporter>("step", false); - registerImporter>("histogram_dist", false); - registerImporter("lognormal_dist", false); - registerImporter("multivariate_normal_dist", false); - registerImporter>("poisson_dist", false); - registerImporter>("poisson_continuous_dist", false); - registerImporter("decay_dist", false); - registerImporter("truth_model_function", false); - registerImporter("delta_resolution_model", false); - registerImporter("gauss_model_function", false); - registerImporter("gauss_resolution_model", false); - registerImporter>("polynomial_dist", false); - registerImporter>("polynomial", false); - registerImporter>("weighted_sum_dist", false); - registerImporter>("weighted_sum", false); - registerImporter("integral", false); - registerImporter("derivative", false); - registerImporter("fft_convolution_dist", false); - registerImporter("extend_pdf", false); - registerImporter("step", false); - registerImporter("spline", false); - - registerExporter(RooWrapperPdf::Class(), "density_function_dist"); - registerExporter>(RooAddPdf::Class(), "mixture_dist", false); - registerExporter>(RooAddModel::Class(), "mixture_model", false); - registerExporter(RooBinSamplingPdf::Class(), "binsampling_dist", false); - registerExporter(RooCrystalBall::Class(), "crystalball_doublesided_dist", false); - registerExporter(RooBinWidthFunction::Class(), "binvolume", false); - registerExporter>(RooLegacyExpPoly::Class(), "legacy_exp_poly_dist", false); - registerExporter(RooExponential::Class(), "exponential_dist", false); - registerExporter>(RooFormulaVar::Class(), "generic", false); - registerExporter>(RooGenericPdf::Class(), "generic_dist", false); - registerExporter>(RooHistFunc::Class(), "step", false); - registerExporter>(RooHistPdf::Class(), "histogram_dist", false); - registerExporter(RooLognormal::Class(), "lognormal_dist", false); - registerExporter(RooGaussian::Class(), "gaussian_dist", true); - registerExporter(RooMultiVarGaussian::Class(), "multivariate_normal_dist", false); - registerExporter(RooPoisson::Class(), "poisson_dist", false); - registerExporter(RooDecay::Class(), "decay_dist", false, /*autoExportDependants=*/false); - registerExporter(RooTruthModel::Class(), "truth_model_function", false); - registerExporter(RooGaussModel::Class(), "gauss_model_function", false); - registerExporter>(RooPolynomial::Class(), "polynomial_dist", false); - registerExporter>(RooPolyVar::Class(), "polynomial", false); - registerExporter>(RooRealSumFunc::Class(), "weighted_sum", false); - registerExporter>(RooRealSumPdf::Class(), "weighted_sum_dist", false); - registerExporter(RooTFnBinding::Class(), "generic", false); - registerExporter(RooRealIntegral::Class(), "integral", false); - registerExporter(RooDerivative::Class(), "derivative", false); - registerExporter(RooFFTConvPdf::Class(), "fft_convolution_dist", false); - registerExporter(RooExtendPdf::Class(), "rate_extended_dist", false); - registerExporter(ParamHistFunc::Class(), "step", false); - registerExporter(RooSpline::Class(), "spline", false); - registerExporter(RooAddition::Class(), "sum", false); - registerExporter(RooArgusBG::Class(), "argus_dist", false); - registerExporter(RooBernstein::Class(), "bernstein_poly_dist", false); - registerExporter(RooBifurGauss::Class(), "bifurkated_gaussian_dist", false); - registerExporter(RooCBShape::Class(), "crystalball_dist", false); - registerExporter(RooChebychev::Class(), "chebychev_dist", false); - registerExporter(RooEffProd::Class(), "efficiency_product_pdf_dist", false); - registerExporter(RooGamma::Class(), "gamma_dist", false); - registerExporter(RooLandau::Class(), "landau_dist", false); - registerExporter(RooPowerSum::Class(), "power_sum_dist", false); - registerExporter(RooProdPdf::Class(), "product_dist", false); - registerExporter(RooProduct::Class(), "product", false); - registerExporter(RooProjectedPdf::Class(), "projected_dist", false); - registerExporter(RooUniform::Class(), "uniform_dist", false); + registerType("argus_dist", importArgus, exportArgus); + registerType("bernstein_poly_dist", importBernstein, exportBernstein); + registerType("bifurkated_gaussian_dist", importBifurGauss, exportBifurGauss); + registerType("binsampling_dist", importBinSamplingPdf, exportBinSamplingPdf); + registerType("binvolume", importBinWidthFunction, exportBinWidthFunction); + registerType("chebychev_dist", importChebychev, exportChebychev); + registerType("crystalball_dist", importCrystalBall, exportCBShape); + registerType("crystalball_doublesided_dist", importCrystalBallDoubleSided, exportCrystalBall); + registerType("decay_dist", importDecay, exportDecay, /*autoExportDependants=*/false); + registerType("density_function_dist", importWrapperPdf, exportWrapperPdf); + registerType("derivative", importDerivative, exportDerivative); + registerType("efficiency_product_pdf_dist", importEffProd, exportEffProd); + registerType("exponential_dist", importExponential, exportExponential); + registerType("fft_convolution_dist", importFFTConvPdf, exportFFTConvPdf); + registerType("gamma_dist", importGamma, exportGamma); + registerType("gauss_model_function", importGaussModel, exportGaussModel); + registerType("gaussian_dist", importGaussian, exportGaussian); + registerType("generic", importFormulaArg, exportFormulaArg); + registerType("generic_dist", importFormulaArg, exportFormulaArg); + registerType("histogram_dist", importHist, exportHist); + registerType("integral", importRealIntegral, exportRealIntegral); + registerType("landau_dist", importLandau, exportLandau); + registerType("legacy_exp_poly_dist", importPolynomial, exportPolynomial); + registerType("lognormal_dist", importLogNormal, exportLogNormal); + registerType("mixture_dist", importAddPdf, exportAddPdf); + registerType("mixture_model", importAddModel, exportAddPdf); + registerType("multivariate_normal_dist", importMultiVarGaussian, exportMultiVarGaussian); + registerType("poisson_dist", importPoisson, exportPoisson); + registerType("polynomial", importPolynomial, exportPolynomial); + registerType("polynomial_dist", importPolynomial, exportPolynomial); + registerType("power_sum_dist", importPowerSum, exportPowerSum); + registerType("product", importProduct, exportProduct); + registerType("product_dist", importProdPdf, exportProdPdf); + registerType("projected_dist", importProjectedPdf, exportProjectedPdf); + registerType("rate_extended_dist", importExtendPdf, exportExtendPdf); + registerType("spline", importSpline, exportSpline); + // A "step" with "data" is a histogram, otherwise it is a ParamHistFunc. + registerType("step", importHist, exportHist); + registerType("step", importParamHistFunc, exportParamHistFunc); + registerType("sum", importAddition, exportAddition); + registerType("truth_model_function", importTruthModel, exportTruthModel); + registerType("uniform_dist", importUniform, exportUniform); + registerType("weighted_sum", importRealSum, exportRealSum); + registerType("weighted_sum_dist", importRealSum, exportRealSum); + + // A RooTFnBinding is written as a generic formula. + registerExporter("generic", exportTFnBinding); + + // The types that are only read: the ones that RooFit writes in another way, and other names of the types above. + registerImporter("array_dist", importArrayDist); + registerImporter("combined_dist", importCombinedDist); + registerImporter("extend_pdf", importExtendPdf); + registerImporter("generic_function", importFormulaArg); + registerImporter("histogram", importHist); + registerImporter("inverse_binvolume", importBinWidthFunction); + registerImporter("normal_dist", importGaussian); + registerImporter("poisson_continuous_dist", importPoisson); + registerDeprecatedImporter("ARGUS_dist", "argus_dist", importArgus); + registerDeprecatedImporter("binsampling", "binsampling_dist", importBinSamplingPdf); + registerDeprecatedImporter("delta_resolution_model", "truth_model_function", importTruthModel); + registerDeprecatedImporter("gauss_resolution_model", "gauss_model_function", importGaussModel); + registerDeprecatedImporter("mixture_resolution_model", "mixture_model", importAddModel); }); } // namespace @@ -2294,7 +1995,7 @@ void writeArrayDist(JSONNode &elem, std::string const &name, std::vector #include #include #include @@ -12,7 +11,6 @@ #include #include -#include #include #include #include @@ -67,59 +65,6 @@ std::string removeSuffix(std::string_view str, std::string_view suffix) return out; } -// Writes a regular binning with "min", "max" and "nbins", and an irregular one with its "edges" only, since HS3 0.3 -// forbids "min" and "max" alongside them. -void writeAxisBinning(JSONNode &node, const RooAbsBinning &binning) -{ - if (binning.isUniform()) { - node["min"] << binning.lowBound(); - node["max"] << binning.highBound(); - node["nbins"] << binning.numBins(); - return; - } - - auto &edges = node["edges"].set_seq(); - edges.append_child() << binning.binLow(0); - for (int i = 0; i < binning.numBins(); ++i) { - edges.append_child() << binning.binHigh(i); - } -} - -// Parse the leading "MAJOR.MINOR" of an HS3 version string like "0.2", "0.2.9" or "0.3". -bool parseHS3Version(std::string const &version, int &major, int &minor) -{ - std::size_t pos = 0; - auto readNumber = [&](int &out) { - const std::size_t begin = pos; - while (pos < version.size() && std::isdigit(static_cast(version[pos]))) { - ++pos; - } - // Reject empty and absurdly long numbers, which std::stoi could not represent. - if (pos == begin || pos - begin > 9) { - return false; - } - out = std::stoi(version.substr(begin, pos - begin)); - return true; - }; - if (!readNumber(major) || pos >= version.size() || version[pos] != '.') { - return false; - } - ++pos; - return readNumber(minor); -} - -// Whether an HS3 version predates `major.minor`. Versions that can't be parsed count as old. That includes the empty -// version of objects that are imported without a document. -bool isHS3VersionBefore(std::string const &version, int major, int minor) -{ - int versionMajor = 0; - int versionMinor = 0; - if (!parseHS3Version(version, versionMajor, versionMinor)) { - return true; - } - return std::make_pair(versionMajor, versionMinor) < std::make_pair(major, minor); -} - namespace { std::vector numBinsPerVariable(RooAbsCollection const &vars) @@ -195,10 +140,10 @@ RooArgList toParamHistFuncOrder(RooArgList const ¶msInBinOrder, RooAbsCollec return out; } -void checkParamHistFuncOrderIsDefined(std::string const &hs3Version, RooAbsCollection const &vars, +void checkParamHistFuncOrderIsDefined(RooFit::HS3::Version const &version, RooAbsCollection const &vars, std::string const &context) { - if (vars.size() >= 2 && isHS3VersionBefore(hs3Version, 0, 3)) { + if (vars.size() >= 2 && version < RooFit::HS3::Version{0, 3}) { RooJSONFactoryWSTool::error(context + " has a multi-dimensional binning. Documents before HS3 0.3 don't define the order " "of its per-bin parameters (RooFit wrote them in TH1 order, while the bin " diff --git a/roofit/hs3/src/JSONIOUtils.h b/roofit/hs3/src/JSONIOUtils.h index b96d556dc2bd3..d7b9b15a17ad6 100644 --- a/roofit/hs3/src/JSONIOUtils.h +++ b/roofit/hs3/src/JSONIOUtils.h @@ -9,12 +9,18 @@ #include #include +#include "HS3Document.h" + class RooAbsArg; -class RooAbsBinning; +class RooAbsCategory; class RooAbsCollection; class RooAbsPdf; class RooAbsReal; class RooArgList; +class RooArgSet; +class RooCategory; +class RooDataHist; +class RooWorkspace; class ParamHistFunc; // A RooProdPdf with this attribute represents an HS3 `array_dist`: a product of one-dimensional distributions of one @@ -22,17 +28,6 @@ class ParamHistFunc; // observable. inline constexpr const char *arrayDistTag = "hs3_array_dist"; -class RooAbsBinning; -class RooAbsCategory; -class RooAbsCollection; -class RooAbsReal; -class RooArgList; -class RooArgSet; -class RooCategory; -class RooDataHist; -class RooWorkspace; -class ParamHistFunc; - // True if `arg` is a RooConstVar whose name is a plain number (i.e. a literal constant). bool isLiteralConstVar(RooAbsArg const &arg); @@ -63,13 +58,6 @@ void writeReference(RooFit::Detail::JSONNode &node, RooAbsArg const &arg); bool startsWith(std::string_view str, std::string_view prefix); bool endsWith(std::string_view str, std::string_view suffix); std::string removeSuffix(std::string_view str, std::string_view suffix); -void writeAxisBinning(RooFit::Detail::JSONNode &node, const RooAbsBinning &binning); - -// The key of a fixed domain axis (HS3 #131): the parameter only takes this value, and RooFit reads it as a -// RooConstVar. -inline constexpr const char *fixedValueKey = "value"; -bool parseHS3Version(std::string const &version, int &major, int &minor); -bool isHS3VersionBefore(std::string const &version, int major, int minor); // A RooHistFunc or RooHistPdf is evaluated with its own observables (`funcObs`), which are mapped by position onto // the variables of its histogram (`histObs`) and need not have the same names, e.g. after a RooCustomizer replaced @@ -85,7 +73,7 @@ std::vector paramHistFuncParamsInBinOrder(ParamHistFunc const &fun RooArgList toParamHistFuncOrder(RooArgList const ¶msInBinOrder, RooAbsCollection const &vars); // Documents before HS3 0.3 wrote the parameters of multi-dimensional ParamHistFuncs in TH1 order. No such documents // are known; they are rejected instead of being guessed. -void checkParamHistFuncOrderIsDefined(std::string const &hs3Version, RooAbsCollection const &vars, +void checkParamHistFuncOrderIsDefined(RooFit::HS3::Version const &version, RooAbsCollection const &vars, std::string const &context); // Selection keys (HS3 "combined" data sets and "combined_dist"): a key is one label per column, and the columns are diff --git a/roofit/hs3/src/JSONInterface.cxx b/roofit/hs3/src/JSONInterface.cxx index 682010cdb8ec6..a854b6d1b20a3 100644 --- a/roofit/hs3/src/JSONInterface.cxx +++ b/roofit/hs3/src/JSONInterface.cxx @@ -178,15 +178,6 @@ JSONNode &JSONNode::operator<<(bool b) return *this; } -const JSONNode &JSONNode::operator>>(std::string &v) const -{ - if (!json(_value).is_string()) { - throwTypeError(_key, "a string", json(_value)); - } - v = json(_value).get(); - return *this; -} - JSONNode &JSONNode::operator[](std::string const &k) { return _storage->makeNode(json(_value)[k], k); @@ -329,11 +320,6 @@ std::optional JSONNode::try_val_bool() const return std::nullopt; } -bool JSONNode::has_key() const -{ - return !_key.empty(); -} - bool JSONNode::has_val() const { return json(_value).is_primitive(); @@ -380,12 +366,6 @@ JSONNode::const_children_view JSONNode::children() const return {const_child_iterator{list.data()}, const_child_iterator{list.data() + list.size()}}; } -std::ostream &operator<<(std::ostream &os, JSONNode const &s) -{ - s.writeJSON(os); - return os; -} - // JSONTree JSONTree::JSONTree(std::unique_ptr storage) : _storage{std::move(storage)} {} diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 45b5a0d99f43c..77a58119f13f0 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -31,7 +31,6 @@ #include "JSONIOUtils.h" #include "HS3Builder.h" -#include "HS3Schema.h" #include "JSONReadRecorder.h" #include @@ -47,6 +46,7 @@ #include #include #include +#include /** \class RooJSONFactoryWSTool \ingroup roofit_dev_docs_hs3 @@ -188,6 +188,34 @@ void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist namespace { +// Append a name to a list that has every name once. +void addUnique(std::vector &names, std::string const &name) +{ + if (std::find(names.begin(), names.end(), name) == names.end()) { + names.push_back(name); + } +} + +template +std::vector namesOf(Coll_t const &objects) +{ + std::vector names; + for (auto const *object : objects) { + names.push_back(object->GetName()); + } + return names; +} + +// Whether the states of a category have these labels, in any order. +bool hasLabels(RooCategory const &cat, std::vector const &labels) +{ + std::set states; + for (auto const &state : cat) { + states.insert(state.first); + } + return states == std::set(labels.begin(), labels.end()); +} + HS3::Binning toBinning(RooAbsBinning const &binning) { if (binning.isUniform()) { @@ -338,11 +366,7 @@ void createVariables(RooWorkspace &ws, std::vector const &domains, RooJSONFactoryWSTool::error("the axis '" + axis->name + "' is categorical, but '" + axis->name + "' already exists and is not a category"); } - std::set existingLabels; - for (auto const &state : *cat) { - existingLabels.insert(state.first); - } - if (existingLabels != std::set(axis->categories.begin(), axis->categories.end())) { + if (!hasLabels(*cat, axis->categories)) { RooJSONFactoryWSTool::error("the categories of the axis '" + axis->name + "' differ from the states of the existing category '" + axis->name + "'"); } @@ -393,12 +417,14 @@ class ImportGuard { } // namespace -void RooFit::HS3::report(Diagnostics const &diagnostics) +void RooFit::HS3::report(Diagnostics const &diagnostics, std::string const &object) { std::string errors; for (Diagnostic const &diagnostic : diagnostics) { - const std::string text = - diagnostic.path.empty() ? diagnostic.message : diagnostic.path + ": " + diagnostic.message; + // A path starts with the name of a component, or with the index of an entry if the object is an array. + const bool needsDot = !object.empty() && !diagnostic.path.empty() && diagnostic.path.front() != '['; + const std::string path = object + (needsDot ? "." : "") + diagnostic.path; + const std::string text = path.empty() ? diagnostic.message : path + ": " + diagnostic.message; if (diagnostic.severity == Diagnostic::Severity::Error) { errors += (errors.empty() ? "" : "\n") + text; } else { @@ -449,21 +475,6 @@ std::string genPrefix(const JSONNode &p, bool trailing_underscore) return prefix; } -// helpers for serializing / deserializing binned datasets -void genIndicesHelper(std::vector> &combinations, std::vector &curr_comb, - const std::vector &vars_numbins, size_t curridx) -{ - if (curridx == vars_numbins.size()) { - // we have filled a combination. Copy it. - combinations.emplace_back(curr_comb); - } else { - for (int i = 0; i < vars_numbins[curridx]; ++i) { - curr_comb[curridx] = i; - ::genIndicesHelper(combinations, curr_comb, vars_numbins, curridx + 1); - } - } -} - // Import string attributes ("dict") and boolean tags ("tags") onto `arg`. void importAttributes(RooAbsArg *arg, HS3::Attributes const &attributes) { @@ -475,75 +486,46 @@ void importAttributes(RooAbsArg *arg, HS3::Attributes const &attributes) } } -void addIfPresent(RooArgSet &out, RooArgSet const *args) -{ - if (args) { - out.add(*args, true); - } -} - -void collectParameterStepWidthCandidatesFromModelConfigs(RooWorkspace const &workspace, RooArgSet &candidates, - RooArgSet &excluded) +// The step widths of the minimization are the errors of the parameters of the ModelConfigs, and of the parameters of +// the top-level pdfs given the data sets. The observables are no parameters. +void exportParameterStepWidths(RooWorkspace const &workspace, std::vector const &pdfs, + std::vector const &data, std::vector &out) { + RooArgSet candidates; + RooArgSet excluded; + auto addIfPresent = [](RooArgSet &set, RooArgSet const *args) { + if (args) { + set.add(*args, true); + } + }; for (TObject *obj : workspace.allGenericObjects()) { - auto const *mc = dynamic_cast(obj); - if (!mc) { - continue; + if (auto const *mc = dynamic_cast(obj)) { + addIfPresent(candidates, mc->GetParametersOfInterest()); + addIfPresent(candidates, mc->GetNuisanceParameters()); + addIfPresent(excluded, mc->GetObservables()); + addIfPresent(excluded, mc->GetGlobalObservables()); + addIfPresent(excluded, mc->GetConditionalObservables()); } - - addIfPresent(candidates, mc->GetParametersOfInterest()); - addIfPresent(candidates, mc->GetNuisanceParameters()); - - addIfPresent(excluded, mc->GetObservables()); - addIfPresent(excluded, mc->GetGlobalObservables()); - addIfPresent(excluded, mc->GetConditionalObservables()); } -} - -void collectParameterStepWidthCandidatesFromPdfs(std::vector const &pdfs, - std::vector const &data, RooArgSet &candidates, - RooArgSet &excluded) -{ for (RooAbsPdf const *pdf : pdfs) { RooArgSet observables; for (RooAbsData const *dataset : data) { std::unique_ptr pdfObs{pdf->getObservables(*dataset->get())}; observables.add(*pdfObs, true); } - - if (observables.empty()) { - continue; + if (!observables.empty()) { + RooArgSet params; + pdf->getParameters(&observables, params); + candidates.add(params, true); + excluded.add(observables, true); } - - RooArgSet params; - pdf->getParameters(&observables, params); - candidates.add(params, true); - excluded.add(observables, true); } -} - -void exportParameterStepWidths(RooWorkspace const &workspace, std::vector const &pdfs, - std::vector const &data, std::vector &out) -{ - RooArgSet candidates; - RooArgSet excluded; - - collectParameterStepWidthCandidatesFromModelConfigs(workspace, candidates, excluded); - collectParameterStepWidthCandidatesFromPdfs(pdfs, data, candidates, excluded); - candidates.sort(); - for (RooAbsArg *arg : candidates) { - if (excluded.find(*arg)) { - continue; - } - auto *var = dynamic_cast(arg); - if (!var || !var->hasError()) { - continue; + if (var && var->hasError() && !excluded.find(*arg)) { + out.push_back({var->GetName(), var->getError()}); } - - out.push_back({var->GetName(), var->getError()}); } } @@ -561,360 +543,12 @@ void importParameterStepWidths(RooWorkspace &workspace, HS3Document const &doc) } } -// Generate all combinations of bin indices for the RooRealVars in `vars`. -std::vector> generateBinIndices(const RooArgSet &vars) +// Declare the bookkeeping columns of the exported document (see HS3::bookkeepingColumns()): the axes that several +// components of a combined data set share, and whose variable in the workspace `ws` is used by nothing else, i.e. has +// no clients and is in no set of a ModelConfig. RooAbsData::split() keeps such a column (e.g. an event number) in +// every sub-data set. +void declareBookkeepingColumns(HS3Document &doc, RooWorkspace const &ws) { - std::vector> combinations; - std::vector vars_numbins; - vars_numbins.reserve(vars.size()); - for (const RooAbsArg *arg : vars) { - // Continuous variables and categories (one bin per state) - vars_numbins.push_back(dynamic_cast(*arg).numBins()); - } - std::vector curr_comb(vars.size()); - ::genIndicesHelper(combinations, curr_comb, vars_numbins, 0); - return combinations; -} - -template -JSONNode const *findRooFitInternal(JSONNode const &node, Keys_t const &...keys) -{ - return node.find("misc", "ROOT_internal", keys...); -} - -// --------------------------------------------------------------------------------------------------------------- -// Distinct variates. HS3 requires the terms of a likelihood, and the components of a combined distribution or data -// set, to have distinct variates: a variate in two of them is one quantity modelled twice (e.g. a constraint that is -// a term of the likelihood and a factor of a channel), or two quantities with one name (e.g. the observable "mass" -// of two channels, which need their own names, "mass_SR" and "mass_CR"). All cases are checked with the same -// function, sharedVariates(), on the document, after the export and before the import. - -bool isName(JSONNode const &node) -{ - return !node.is_container() && !node.is_null() && !node.is_number(); -} - -// Add the names in `node` (one name or a list) to `out`. -void addNames(JSONNode const &node, std::set &out) -{ - if (node.is_seq()) { - for (JSONNode const &child : node.children()) { - addNames(child, out); - } - } else if (isName(node)) { - out.insert(node.val()); - } -} - -// The child `name` of the section `section` of the document, or nullptr. -JSONNode const *findInSection(JSONNode const &rootnode, const char *section, std::string const &name) -{ - JSONNode const *node = rootnode.find(section); - return node ? RooJSONFactoryWSTool::findNamedChild(*node, name) : nullptr; -} - -std::string typeOf(JSONNode const &node) -{ - return node.has_child("type") ? node["type"].val() : ""; -} - -// The names of the components of a combined distribution or data set, written as {"key": ..., : }. -std::vector combinedComponents(JSONNode const &node, const char *field) -{ - std::vector out; - if (JSONNode const *components = node.find("components")) { - for (JSONNode const &component : components->children()) { - if (JSONNode const *name = component.find(field); name && isName(*name)) { - out.push_back(name->val()); - } - } - } - return out; -} - -// The variates of the distribution `name` of the document: the "x" of a fundamental distribution (a name or a list) -// and the axes of a distribution with axes (e.g. histfactory_dist), through products, mixtures and combined -// distributions, and the "x" of the components of an array distribution. The constraint terms named by a -// histfactory_dist are terms of their own, see likelihoodTerms(). -void collectVariates(JSONNode const &rootnode, std::string const &name, std::set &out, - std::set &visited) -{ - if (!visited.insert(name).second) { - return; - } - JSONNode const *node = findInSection(rootnode, "distributions", name); - if (!node) { - return; - } - std::string const type = typeOf(*node); - auto recurse = [&](std::vector const &parts) { - for (std::string const &part : parts) { - collectVariates(rootnode, part, out, visited); - } - }; - auto namesIn = [&](const char *key) { - std::vector parts; - if (JSONNode const *list = node->find(key)) { - for (JSONNode const &part : list->children()) { - if (isName(part)) { - parts.push_back(part.val()); - } - } - } - return parts; - }; - if (type == "product_dist") { - recurse(namesIn("factors")); - } else if (type == "mixture_dist") { - recurse(namesIn("summands")); - } else if (type == "combined_dist") { - recurse(combinedComponents(*node, "dist")); - } else if (type == "array_dist") { - if (JSONNode const *components = node->find("components")) { - for (JSONNode const &component : components->children()) { - if (JSONNode const *x = component.find("x")) { - addNames(*x, out); - } - } - } - } else { - if (JSONNode const *x = node->find("x")) { - addNames(*x, out); - } - if (JSONNode const *axes = node->find("axes")) { - for (JSONNode const &axis : axes->children()) { - if (JSONNode const *axisName = axis.find("name")) { - addNames(*axisName, out); - } - } - } - } -} - -std::set variatesOf(JSONNode const &rootnode, std::string const &name) -{ - std::set variates; - std::set visited; - collectVariates(rootnode, name, variates, visited); - return variates; -} - -// The terms of the likelihood `likelihood` of the document, each once: its distributions (the components of a -// combined distribution, or of a RooSimultaneous written to ROOT_internal by older versions, are terms of their own), -// its "aux_distributions", and the constraint terms named by the modifiers of its histfactory_dist terms. -std::vector likelihoodTerms(JSONNode const &rootnode, JSONNode const &likelihood) -{ - std::vector terms; - auto add = [&](std::string const &term) { - if (std::find(terms.begin(), terms.end(), term) == terms.end()) { - terms.push_back(term); - } - }; - for (const char *key : {"distributions", "aux_distributions"}) { - JSONNode const *list = likelihood.find(key); - if (!list) { - continue; - } - for (JSONNode const &item : list->children()) { - if (!isName(item)) { - continue; - } - std::string const name = item.val(); - if (JSONNode const *node = findInSection(rootnode, "distributions", name)) { - if (typeOf(*node) == "combined_dist") { - for (std::string const &component : combinedComponents(*node, "dist")) { - add(component); - } - continue; - } - } else if (JSONNode const *legacy = findRooFitInternal(rootnode, "combined_distributions", name)) { - if (JSONNode const *components = legacy->find("distributions")) { - for (JSONNode const &component : components->children()) { - add(component.val()); - } - continue; - } - } - add(name); - } - } - for (std::size_t i = 0; i < terms.size(); ++i) { - JSONNode const *node = findInSection(rootnode, "distributions", terms[i]); - if (!node || typeOf(*node) != "histfactory_dist" || !node->has_child("samples")) { - continue; - } - for (JSONNode const &sample : (*node)["samples"].children()) { - if (JSONNode const *modifiers = sample.find("modifiers")) { - for (JSONNode const &modifier : modifiers->children()) { - if (JSONNode const *constraints = modifier.find("constraints")) { - std::set names; - addNames(*constraints, names); - for (std::string const &constraint : names) { - add(constraint); - } - } - } - } - } - } - return terms; -} - -// The parts (terms or components) of something that must have distinct variates, with their variates. -using PartVariates = std::vector>>; - -// The variates that several of `parts` share, one message each. `what` names the parts and `context` the whole, -// e.g. "terms" of "the likelihood 'lh'". -std::vector sharedVariates(PartVariates const &parts, std::string const &what, std::string const &context) -{ - std::map> partsOfVariate; - std::vector order; - for (auto const &[part, variates] : parts) { - for (std::string const &variate : variates) { - auto &list = partsOfVariate[variate]; - if (list.empty()) { - order.push_back(variate); - } - if (std::find(list.begin(), list.end(), part) == list.end()) { - list.push_back(part); - } - } - } - std::vector messages; - for (std::string const &variate : order) { - auto const &list = partsOfVariate[variate]; - if (list.size() < 2) { - continue; - } - std::string names; - for (std::size_t i = 0; i < list.size(); ++i) { - names += (i == 0 ? "'" : (i + 1 == list.size() ? "' and '" : "', '")) + list[i]; - } - messages.push_back("the variate '" + variate + "' is shared by the " + what + " " + names + "' of " + context); - } - return messages; -} - -// The axes of the components of a combined data set, by component. -PartVariates combinedDataAxes(JSONNode const &rootnode, JSONNode const &dataset) -{ - PartVariates parts; - for (std::string const &component : combinedComponents(dataset, "dataset")) { - std::set axes; - if (JSONNode const *node = findInSection(rootnode, "data", component)) { - if (JSONNode const *axesNode = node->find("axes")) { - for (JSONNode const &axis : axesNode->children()) { - if (JSONNode const *axisName = axis.find("name")) { - addNames(*axisName, axes); - } - } - } - } - parts.emplace_back(component, axes); - } - return parts; -} - -// Bookkeeping columns (a RooFit extension, in misc.ROOT_internal.bookkeeping_columns as {"": -// [, ...]}): axes that several components of a combined data set share, but that are no variate of any -// distribution, e.g. an event number. They have no statistical meaning, and are exempt from the requirement of -// distinct axes. A declared column that is a variate of a distribution is not a bookkeeping column. -std::map> bookkeepingColumnsOfDocument(JSONNode const &rootnode) -{ - std::map> out; - JSONNode const *declared = findRooFitInternal(rootnode, "bookkeeping_columns"); - if (!declared || !declared->is_map()) { - return out; - } - std::set variates; - if (JSONNode const *distributions = rootnode.find("distributions")) { - for (JSONNode const &dist : distributions->children()) { - std::set const v = variatesOf(rootnode, RooJSONFactoryWSTool::name(dist)); - variates.insert(v.begin(), v.end()); - } - } - for (JSONNode const &entry : declared->children()) { - for (JSONNode const &column : entry.children()) { - if (!variates.count(column.val())) { - out[entry.key()].insert(column.val()); - } - } - } - return out; -} - -// All shared variates of the document: of the terms of each likelihood, of the components of the combined -// distributions that no likelihood lists (the others are checked through the likelihood), and the shared axes of the -// components of all combined data sets, except for bookkeeping columns. -std::vector sharedVariatesOfDocument(JSONNode const &rootnode) -{ - std::vector messages; - auto append = [&](std::vector const &more) { - messages.insert(messages.end(), more.begin(), more.end()); - }; - std::set inLikelihoods; - if (JSONNode const *likelihoods = rootnode.find("likelihoods")) { - for (JSONNode const &likelihood : likelihoods->children()) { - for (const char *key : {"distributions", "data", "aux_distributions"}) { - if (JSONNode const *list = likelihood.find(key)) { - for (JSONNode const &item : list->children()) { - inLikelihoods.insert(item.val()); - } - } - } - PartVariates parts; - for (std::string const &term : likelihoodTerms(rootnode, likelihood)) { - parts.emplace_back(term, variatesOf(rootnode, term)); - } - append(sharedVariates(parts, "terms", "the likelihood '" + RooJSONFactoryWSTool::name(likelihood) + "'")); - } - } - if (JSONNode const *distributions = rootnode.find("distributions")) { - for (JSONNode const &dist : distributions->children()) { - std::string const name = RooJSONFactoryWSTool::name(dist); - if (typeOf(dist) != "combined_dist" || inLikelihoods.count(name)) { - continue; - } - PartVariates parts; - for (std::string const &component : combinedComponents(dist, "dist")) { - parts.emplace_back(component, variatesOf(rootnode, component)); - } - append(sharedVariates(parts, "components", "the combined_dist '" + name + "'")); - } - } - // The axes of the components of every combined data set, also of those that a likelihood lists, except for the - // bookkeeping columns. - std::map> const bookkeeping = bookkeepingColumnsOfDocument(rootnode); - if (JSONNode const *data = rootnode.find("data")) { - for (JSONNode const &dataset : data->children()) { - std::string const name = RooJSONFactoryWSTool::name(dataset); - if (typeOf(dataset) != "combined") { - continue; - } - PartVariates parts = combinedDataAxes(rootnode, dataset); - if (auto found = bookkeeping.find(name); found != bookkeeping.end()) { - for (auto &part : parts) { - for (std::string const &column : found->second) { - part.second.erase(column); - } - } - } - append(sharedVariates(parts, "components", "the combined data set '" + name + "'")); - } - } - return messages; -} - -// Declare the bookkeeping columns of the exported document `rootnode` (see bookkeepingColumnsOfDocument()): the axes -// that several components of a combined data set share, and whose variable in the workspace `ws` is used by nothing -// else, i.e. has no clients and is in no set of a ModelConfig. RooAbsData::split() keeps such a column (e.g. an event -// number) in every sub-data set. -void declareBookkeepingColumns(JSONNode &rootnode, RooWorkspace const &ws) -{ - JSONNode const *data = rootnode.find("data"); - if (!data) { - return; - } std::vector modelConfigSets; for (TObject *obj : ws.allGenericObjects()) { if (auto *mc = dynamic_cast(obj)) { @@ -934,117 +568,14 @@ void declareBookkeepingColumns(JSONNode &rootnode, RooWorkspace const &ws) return std::none_of(modelConfigSets.begin(), modelConfigSets.end(), [&](RooArgSet const *set) { return set->find(name.c_str()) != nullptr; }); }; - std::map> columns; - for (JSONNode const &dataset : data->children()) { - if (typeOf(dataset) != "combined") { - continue; - } - std::map counts; - for (auto const &[component, axes] : combinedDataAxes(rootnode, dataset)) { - for (std::string const &axis : axes) { - ++counts[axis]; - } - } - for (auto const &[axis, count] : counts) { - if (count > 1 && usedByNothing(axis)) { - columns[RooJSONFactoryWSTool::name(dataset)].insert(axis); - } - } - } - if (columns.empty()) { - return; - } - JSONNode &node = rootnode["misc"]["ROOT_internal"]["bookkeeping_columns"].set_map(); - for (auto const &[dataset, names] : columns) { - node[dataset].fill_seq(names); - } -} - -// Write each term of the likelihoods of the exported document `rootnode` once. A RooFit model often has a constraint -// both as a factor of the product of a channel and as a term of the likelihood (e.g. a constraint of a HistFactory -// modifier, an external constraint of the ModelConfig, or any constraint of a channel, see -// constraintsOfChannels()); RooFit counts it once. In HS3 it would enter twice, so the factor is removed from -// the product. Only products that no other distribution uses are changed (being a component -// of a combined distribution is not a use), and only factors that are terms of every likelihood that has the product -// as a term. -void writeLikelihoodTermsOnce(JSONNode &rootnode) -{ - JSONNode const *likelihoods = rootnode.find("likelihoods"); - JSONNode const *distributions = rootnode.find("distributions"); - if (!likelihoods || !distributions) { - return; - } - // The products that are terms, with the factors that are terms wherever the product is one. - std::map> removable; - for (JSONNode const &likelihood : likelihoods->children()) { - std::vector const terms = likelihoodTerms(rootnode, likelihood); - for (std::string const &term : terms) { - JSONNode const *node = findInSection(rootnode, "distributions", term); - if (!node || typeOf(*node) != "product_dist" || !node->has_child("factors")) { - continue; - } - std::set factorsThatAreTerms; - for (JSONNode const &factor : (*node)["factors"].children()) { - if (factor.val() != term && std::find(terms.begin(), terms.end(), factor.val()) != terms.end()) { - factorsThatAreTerms.insert(factor.val()); - } - } - auto found = removable.find(term); - if (found == removable.end()) { - removable[term] = factorsThatAreTerms; - } else { - std::set both; - std::set_intersection(found->second.begin(), found->second.end(), factorsThatAreTerms.begin(), - factorsThatAreTerms.end(), std::inserter(both, both.begin())); - found->second = both; - } - } - } - // Products that another distribution uses are left alone. - for (JSONNode const &dist : distributions->children()) { - std::vector used; - for (const char *key : {"factors", "summands"}) { - if (JSONNode const *parts = dist.find(key)) { - for (JSONNode const &part : parts->children()) { - if (isName(part)) { - used.push_back(part.val()); - } - } - } - } - if (typeOf(dist) != "combined_dist") { - for (std::string const &component : combinedComponents(dist, "dist")) { - used.push_back(component); - } - } - for (std::string const &name : used) { - removable.erase(name); - } - } - for (auto const &[product, remove] : removable) { - if (remove.empty()) { - continue; - } - auto &node = const_cast(*findInSection(rootnode, "distributions", product)); - std::vector keep; - for (JSONNode const &factor : node["factors"].children()) { - if (remove.find(factor.val()) == remove.end()) { - keep.push_back(factor.val()); + for (auto const &[dataset, axes] : HS3::sharedAxes(doc)) { + for (std::string const &axis : axes) { + if (usedByNothing(axis)) { + doc.rooFit.bookkeepingColumns[dataset].push_back(axis); } } - if (keep.empty()) { - continue; - } - for (std::string const &factor : remove) { - oocoutI(nullptr, IO) << "RooFitHS3: '" << factor << "' is a factor of '" << product - << "' and a term of its likelihood; it is written once, as a term of the likelihood" - << std::endl; - } - node["factors"].clear(); - node["factors"].fill_seq(keep); } } -// --------------------------------------------------------------------------------------------------------------- // Export the string attributes and tags of `arg` into the ROOT-internal attributes. void exportAttributes(const RooAbsArg *arg, std::map &attributes) @@ -1083,13 +614,12 @@ void exportAttributes(const RooAbsArg *arg, std::map createObservable(HS3::BinnedAxis const &axis) +// A categorical axis has the labels in the order of the states of the category, which is the order of the bins of a +// RooDataHist. +HS3::CategoricalAxis toCategoricalAxis(RooAbsCategory const &cat) { - auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); - applyBinning(*obs, axis.binning); - return obs; + HS3::CategoricalAxis axis; + axis.name = cat.GetName(); + for (unsigned int i = 0; i < cat.size(); ++i) { + axis.categories.push_back(cat.getOrdinal(i).first); + } + return axis; } -// The labels of a category, as a set. -std::set labelsOf(RooCategory const &cat) +// Binned data over variables that are RooRealVars or RooCategories. +HS3::BinnedData toBinnedData(RooArgSet const &vars, std::size_t n, double const *contents) { - std::set labels; - for (auto const &state : cat) { - labels.insert(state.first); + HS3::BinnedData data; + for (RooAbsArg const *var : vars) { + if (auto const *cat = dynamic_cast(var)) { + RooJSONFactoryWSTool::testValidName(cat->GetName(), false); + data.axes.emplace_back(toCategoricalAxis(*cat)); + } else { + data.axes.emplace_back(HS3::toBinnedAxis(static_cast(*var))); + } } - return labels; + data.contents.assign(contents, contents + n); + return data; +} + +std::unique_ptr createObservable(HS3::BinnedAxis const &axis) +{ + auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); + applyBinning(*obs, axis.binning); + return obs; } // Create the variables for the axes of binned data. A categorical axis is one bin per category. HS3 identifies @@ -1143,7 +691,7 @@ RooArgSet createBinnedObservables(std::vector(axisVariant); auto cat = std::make_unique(axis.name.c_str(), axis.name.c_str()); if (RooCategory const *wsCat = workspace.cat(axis.name)) { - if (labelsOf(*wsCat) != std::set(axis.categories.begin(), axis.categories.end())) { + if (!hasLabels(*wsCat, axis.categories)) { RooJSONFactoryWSTool::error("the categories of the axis '" + axis.name + "' of the dataset '" + dataName + "' differ from those of the category '" + axis.name + "'"); } @@ -1237,7 +785,7 @@ std::unique_ptr createUnbinnedData(std::string const &name, HS3::Unb "\" could not be found in the workspace! Categorical observables are " "declared by a categorical axis in a domain."); } - if (labelsOf(*cat) != std::set(catAxis.categories.begin(), catAxis.categories.end())) { + if (!hasLabels(*cat, catAxis.categories)) { RooJSONFactoryWSTool::error("The categories of the axis \"" + catAxis.name + "\" differ from the states of the category \"" + catAxis.name + "\""); } @@ -1324,10 +872,7 @@ void requirePdf(RooAbsArg const *arg, std::string const &what) void checkCombinedPairing(RooSimultaneous const &simPdf, std::string const &dataName, HS3::CombinedData const &data) { std::vector columns = selectionColumns(simPdf.indexCat()); - std::vector columnNames; - for (RooCategory const *column : columns) { - columnNames.push_back(column->GetName()); - } + const std::vector columnNames = namesOf(columns); std::vector dataColumns = data.columns; std::sort(dataColumns.begin(), dataColumns.end()); if (dataColumns != columnNames) { @@ -1385,15 +930,14 @@ void checkCombinedPairing(RooSimultaneous const &simPdf, std::string const &data } } -// Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. // The observables of the analysis `analysis` if no data set of its likelihood gives them: the list that RooFit writes // in misc.ROOT_internal.ModelConfigs for a ModelConfig without data, or else the variables that HS3 makes // observables. By the standard, all parameters of the distributions of the likelihood are listed in a domain of the -// analysis or are constant, so a variate of the distributions of the likelihood (`variates`, see collectVariates()) -// that is neither, is an observable. Requiring all three excludes the parameters, and the observed values of the -// constraint terms, which are constant parameters. -void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysis, HS3::ModelConfigInfo const *mcInfo, - std::set const &variates, RooWorkspace &workspace, RooArgSet &observables) +// analysis or are constant, so a variate of the distributions of the likelihood (see HS3::variatesOf(), the columns +// of a combined distribution are observables as well) that is neither, is an observable. Requiring all three excludes +// the parameters, and the observed values of the constraint terms, which are constant parameters. +void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysis, HS3::Likelihood const &likelihood, + HS3::ModelConfigInfo const *mcInfo, RooWorkspace &workspace, RooArgSet &observables) { if (mcInfo && mcInfo->observables) { for (std::string const &name : *mcInfo->observables) { @@ -1413,7 +957,7 @@ void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysi continue; } for (auto const &axis : domain.axes) { - inDomains.insert(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); + inDomains.insert(HS3::nameOf(axis)); } } } @@ -1421,7 +965,7 @@ void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysi inDomains.insert(poi.name); } std::string list; - for (std::string const &variate : variates) { + for (std::string const &variate : HS3::variatesOf(doc, likelihood)) { RooAbsArg *arg = workspace.arg(variate); if (!dynamic_cast(arg) && !dynamic_cast(arg)) { continue; @@ -1440,9 +984,9 @@ void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysi } } +// Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWorkspace &workspace, - std::vector> &datasets, std::set const &bookkeepingColumns, - std::set const &variates) + std::vector> &datasets, std::set const &bookkeepingColumns) { // if this is a toplevel pdf, also create a modelConfig for it std::string const &analysisName = analysis.name; @@ -1620,7 +1164,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo } } if (observables.empty()) { - observablesWithoutData(doc, analysis, mcInfo, variates, workspace, observables); + observablesWithoutData(doc, analysis, likelihood, mcInfo, workspace, observables); } // A combined distribution paired with a combined data set must have a sub-data set for every component. @@ -1718,7 +1262,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo if (thisDomain == doc.domains.end()) continue; for (auto const &axis : thisDomain->axes) { - auto *wsvar = workspace.var(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); + auto *wsvar = workspace.var(HS3::nameOf(axis)); if (wsvar) domainPars.add(*wsvar); } @@ -1986,10 +1530,14 @@ std::unique_ptr RooFit::HS3::createDataHist(std::string const &name RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + "': errors=" + std::to_string(errors->size()) + ", contents=" + std::to_string(contents.size())); } - auto bins = generateBinIndices(vars); - if (contents.size() != bins.size()) { + // A continuous variable has the bins of its binning, and a category one bin per state. + std::size_t nBins = 1; + for (RooAbsArg const *var : vars) { + nBins *= std::max(0, dynamic_cast(*var).numBins()); + } + if (contents.size() != nBins) { RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.size()) + - ", bins=" + std::to_string(bins.size())); + ", bins=" + std::to_string(nBins)); } auto dh = std::make_unique(name, name, vars); // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from @@ -1997,7 +1545,7 @@ std::unique_ptr RooFit::HS3::createDataHist(std::string const &name if (errors && !errors->empty()) { dh->set(0, 0., 1.); } - for (std::size_t ibin = 0; ibin < bins.size(); ++ibin) { + for (std::size_t ibin = 0; ibin < nBins; ++ibin) { dh->set(ibin, contents[ibin], errors ? (*errors)[ibin] : -1); } return dh; @@ -2007,6 +1555,11 @@ RooJSONFactoryWSTool::RooJSONFactoryWSTool(RooWorkspace &ws) : _workspace{ws} {} RooJSONFactoryWSTool::~RooJSONFactoryWSTool() = default; +HS3::Version RooJSONFactoryWSTool::hs3Version() const +{ + return _import ? _import->document.metadata.version : HS3::Version{}; +} + void RooJSONFactoryWSTool::fillSeq(JSONNode &node, RooAbsCollection const &coll, size_t nMax) { const size_t old_children = node.num_children(); @@ -2042,32 +1595,12 @@ JSONNode const *RooJSONFactoryWSTool::findNamedChild(JSONNode const &node, std:: return nullptr; } -/** - * @brief Check if a string is a valid name. - * - * A valid name should start with a letter or an underscore, followed by letters, digits, or underscores. - * Only characters from the ASCII character set are allowed. - * - * @param str The string to be checked. - * @return bool Returns true if the string is a valid name; otherwise, returns false. - */ +/// Whether a string is a valid name: a letter or an underscore, followed by letters, digits or underscores. Only +/// characters from the ASCII character set are allowed. bool RooJSONFactoryWSTool::isValidName(const std::string &str) { - // Check if the string is empty or starts with a non-letter/non-underscore character - if (str.empty() || !(std::isalpha(str[0]) || str[0] == '_')) { - return false; - } - - // Check the remaining characters in the string - for (char c : str) { - // Allow letters, digits, and underscore - if (!(std::isalnum(c) || c == '_')) { - return false; - } - } - - // If all characters are valid, the string is a valid name - return true; + return !str.empty() && (std::isalpha(str[0]) || str[0] == '_') && + std::all_of(str.begin(), str.end(), [](char c) { return std::isalnum(c) || c == '_'; }); } bool RooJSONFactoryWSTool::testValidName(const std::string &name, bool forceError) @@ -2091,11 +1624,6 @@ std::string RooJSONFactoryWSTool::name(const JSONNode &n) return n["name"].val(); } -JSONNode &RooJSONFactoryWSTool::makeVariablesNode(JSONNode &rootNode) -{ - return appendNamedChild(rootNode["parameter_points"], "default_values")["parameters"]; -} - template <> RooRealVar *RooJSONFactoryWSTool::requestImpl(const std::string &objname) { @@ -2245,16 +1773,6 @@ std::string selectionKeyName(std::vector const &key) return out; } -// The names of the category columns of `vars`. -std::vector selectionColumnNames(RooAbsCollection const &vars) -{ - std::vector names; - for (RooCategory const *column : selectionColumns(vars)) { - names.push_back(column->GetName()); - } - return names; -} - // The RooSimultaneous of the workspace (the first by name) whose columns are `columns`, or nullptr. RooSimultaneous const *findMatchingSimultaneous(RooWorkspace const &workspace, std::vector const &columns) { @@ -2266,11 +1784,7 @@ RooSimultaneous const *findMatchingSimultaneous(RooWorkspace const &workspace, s } sortByName(candidates); for (RooSimultaneous const *candidate : candidates) { - std::vector candidateColumns; - for (RooCategory const *column : selectionColumns(candidate->indexCat())) { - candidateColumns.push_back(column->GetName()); - } - if (candidateColumns == columns) { + if (namesOf(selectionColumns(candidate->indexCat())) == columns) { return candidate; } } @@ -2281,7 +1795,8 @@ RooSimultaneous const *findMatchingSimultaneous(RooWorkspace const &workspace, s namespace { -// Record an object that was written by an exporter in the document. +// Record an object that was written by an exporter in the document. Its node moves when the array that it is in +// grows, so the distributions only get their nodes at the end of the export. void recordGenericObject(HS3Document &doc, bool isPdf, std::string const &name, JSONNode const &elem) { std::string type = elem.has_child("type") ? elem["type"].val() : ""; @@ -2483,9 +1998,7 @@ void RooJSONFactoryWSTool::recordConstraintTerms(std::string const &pdfName, std { auto &out = _export->constraintTermsOfPdf[pdfName]; for (auto const &term : terms) { - if (std::find(out.begin(), out.end(), term) == out.end()) { - out.push_back(term); - } + addUnique(out, term); } } @@ -2608,11 +2121,7 @@ void RooJSONFactoryWSTool::exportSimultaneous(RooSimultaneous const &simPdf) // document is read back. void RooJSONFactoryWSTool::recordCategory(RooAbsCategory const &cat) { - HS3::CategoricalAxis axis; - axis.name = cat.GetName(); - for (unsigned int i = 0; i < cat.size(); ++i) { - axis.categories.push_back(cat.getOrdinal(i).first); - } + HS3::CategoricalAxis axis = toCategoricalAxis(cat); auto found = _export->categories.find(axis.name); if (found != _export->categories.end() && found->second.categories != axis.categories) { RooJSONFactoryWSTool::error("the category '" + axis.name + @@ -2621,16 +2130,9 @@ void RooJSONFactoryWSTool::recordCategory(RooAbsCategory const &cat) _export->categories[axis.name] = std::move(axis); } -/** - * @brief Import a function from the JSONNode into the workspace. - * - * This function imports a function from the given JSONNode into the workspace. - * The function's information is read from the JSONNode and added to the workspace. - * - * @param p The JSONNode representing the function to be imported. - * @param importAllDependants A boolean flag indicating whether to import all dependants (servers) of the function. - * @return void - */ +/// Import the function or distribution `p` into the workspace, with the importers that are registered for its type. +/// If `importAllDependants` is false, `p` is a fragment of a document, whose own "functions" and "distributions" are +/// imported first. void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDependants) { // To report the components that no importer understands. @@ -2725,99 +2227,24 @@ void RooJSONFactoryWSTool::importDistribution(HS3::Distribution const &distribut found != channelConstraints.end() ? &found->second : nullptr); } -/** - * @brief Import a function from a JSON string into the workspace. - * - * This function imports a function from the provided JSON string into the workspace. - * The function's information is read from the JSON string and added to the workspace. - * - * @param jsonString The JSON string containing the function information. - * @param importAllDependants A boolean flag indicating whether to import all dependants (servers) of the function. - * @return void - */ +/// Import a function or distribution that is given as a JSON string, see the overload for a JSONNode. void RooJSONFactoryWSTool::importFunction(const std::string &jsonString, bool importAllDependants) { this->importFunction((JSONTree::create(jsonString))->rootnode(), importAllDependants); } -/** - * @brief Export the name and binning of a RooRealVar to a JSONNode. - * - * @param obsNode The JSONNode to which the axis information will be exported. - * @param var The RooRealVar representing the axis to be exported. - * @return void - */ -// A categorical axis: the labels in the order of the states of the category, which is the order of the bins of a -// RooDataHist. -void RooJSONFactoryWSTool::exportCategoricalAxis(JSONNode &obsNode, RooCategory const &cat) -{ - testValidName(cat.GetName(), false); - obsNode["name"] << cat.GetName(); - auto &labels = obsNode["categories"].set_seq(); - for (unsigned int i = 0; i < cat.size(); ++i) { - labels.append_child() << cat.getOrdinal(i).first; - } -} - -void RooJSONFactoryWSTool::exportAxis(JSONNode &obsNode, RooRealVar const &var) -{ - std::string name = var.GetName(); - RooJSONFactoryWSTool::testValidName(name, false); - obsNode["name"] << name; - - writeAxisBinning(obsNode, var.getBinning()); -} - -/** - * @brief Export histogram data to a JSONNode. - * - * This function exports histogram data, represented by the provided variables and contents, to a JSONNode. - * The histogram's axes information and bin contents are added as key-value pairs to the JSONNode. - * - * @param vars The RooArgSet representing the variables associated with the histogram. - * @param n The number of bins in the histogram. - * @param contents A pointer to the array containing the bin contents of the histogram. - * @param output The JSONNode to which the histogram data will be exported. - * @return void - */ +/// Write the axes and the `n` bin contents of a histogram over the variables `vars`, which are RooRealVars or +/// RooCategories, in the order of the variables. void RooJSONFactoryWSTool::exportHisto(RooArgSet const &vars, std::size_t n, double const *contents, JSONNode &output) { - auto &observablesNode = output["axes"].set_seq(); - // axes have to be ordered to get consistent bin indices - for (RooAbsArg const *var : vars) { - if (auto const *cat = dynamic_cast(var)) { - exportCategoricalAxis(observablesNode.append_child().set_map(), *cat); - } else { - exportAxis(observablesNode.append_child().set_map(), static_cast(*var)); - } - } - - return exportArray(n, contents, output["contents"]); + HS3::writeBinnedData(output, toBinnedData(vars, n, contents)); } -/** - * @brief Export an array of doubles to a JSONNode. - * - * This function exports an array of doubles, represented by the provided size and contents, - * to a JSONNode. The array elements are added to the JSONNode as a sequence of values. - * - * @param n The size of the array. - * @param contents A pointer to the array containing the double values. - * @param output The JSONNode to which the array will be exported. - * @return void - */ +/// Write `n` numbers as an array, each as an integer if it has no fractional part. RooFit doesn't use this function +/// itself, but exporters outside of ROOT do. void RooJSONFactoryWSTool::exportArray(std::size_t n, double const *contents, JSONNode &output) { - output.set_seq(); - for (std::size_t i = 0; i < n; ++i) { - double w = contents[i]; - // To make sure there are no unnecessary floating points in the JSON - if (int(w) == w) { - output.append_child() << int(w); - } else { - output.append_child() << w; - } - } + HS3::writeContents(output, std::vector(contents, contents + n)); } // Config::renameCombinedObservables: work out, before anything is exported, the new names of the observables of @@ -2842,7 +2269,7 @@ void RooJSONFactoryWSTool::prepareCombinedObservableRenaming() if (columns.empty()) { continue; } - RooSimultaneous const *simPdf = findMatchingSimultaneous(_workspace, selectionColumnNames(*data->get())); + RooSimultaneous const *simPdf = findMatchingSimultaneous(_workspace, namesOf(selectionColumns(*data->get()))); for (std::vector const &key : selectionKeys(columns)) { const std::string keyName = selectionKeyName(key); RooArgSet candidates; @@ -3000,31 +2427,18 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) variables.remove(*weightVar); } - // Categories are categorical axes, declared by a categorical axis of the default domain. - auto categoricalAxis = [&](RooCategory const &cat) { - recordCategory(cat); - HS3::CategoricalAxis axis; - axis.name = cat.GetName(); - testValidName(axis.name, false); - for (unsigned int i = 0; i < cat.size(); ++i) { - axis.categories.push_back(cat.getOrdinal(i).first); + // The ranges of the variables are written to the domains. Categories are declared by a categorical axis of the + // default domain. + for (RooAbsArg const *arg : variables) { + if (auto const *cat = dynamic_cast(arg)) { + recordCategory(*cat); + } else { + recordDomains(_export->domains, static_cast(*arg)); } - return axis; - }; + } auto exportBinned = [&](std::size_t n, double const *contents) { - HS3::BinnedData binned; - for (RooAbsArg const *arg : variables) { - if (auto const *cat = dynamic_cast(arg)) { - binned.axes.emplace_back(categoricalAxis(*cat)); - continue; - } - auto const *var = static_cast(arg); - recordDomains(_export->domains, *var); - binned.axes.emplace_back(HS3::toBinnedAxis(*var)); - } - binned.contents.assign(contents, contents + n); - output.body = std::move(binned); + output.body = toBinnedData(variables, n, contents); _export->doc.data.push_back(std::move(output)); }; @@ -3064,15 +2478,14 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) // this really is an unbinned dataset HS3::UnbinnedData unbinned; for (RooAbsArg const *arg : variables) { + testValidName(arg->GetName(), false); if (auto const *cat = dynamic_cast(arg)) { - unbinned.axes.emplace_back(categoricalAxis(*cat)); - continue; + unbinned.axes.emplace_back(toCategoricalAxis(*cat)); + } else { + // HS3 forbids binning information on the axes of unbinned data. + auto const &var = static_cast(*arg); + unbinned.axes.emplace_back(HS3::ContinuousAxis{var.GetName(), var.getMin(), var.getMax()}); } - auto const *var = static_cast(arg); - recordDomains(_export->domains, *var); - // HS3 forbids binning information on the axes of unbinned data. - testValidName(var->GetName(), false); - unbinned.axes.emplace_back(HS3::ContinuousAxis{var->GetName(), var->getMin(), var->getMax()}); } std::vector weightVals; bool hasNonUnityWeights = false; @@ -3099,15 +2512,7 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) _export->doc.data.push_back(std::move(output)); } -/** - * @brief Read axes from the JSONNode and create a RooArgSet representing them. - * - * This function reads axes information from the given JSONNode and - * creates a RooArgSet with variables representing these axes. - * - * @param topNode The JSONNode containing the axes information to be read. - * @return RooArgSet A RooArgSet containing the variables created from the JSONNode. - */ +/// Create the variables for the "axes" of `topNode`, which are owned by the returned set. RooArgSet RooJSONFactoryWSTool::readAxes(const JSONNode &topNode) { HS3::Diagnostics diagnostics; @@ -3116,17 +2521,8 @@ RooArgSet RooJSONFactoryWSTool::readAxes(const JSONNode &topNode) return HS3::createObservables(axes); } -/** - * @brief Read binned data from the JSONNode and create a RooDataHist object. - * - * This function reads binned data from the given JSONNode and creates a RooDataHist object. - * The binned data is associated with the specified name and variables (RooArgSet) in the workspace. - * - * @param n The JSONNode representing the binned data to be read. - * @param name The name to be associated with the created RooDataHist object. - * @param vars The RooArgSet representing the variables associated with the binned data. - * @return std::unique_ptr A unique pointer to the created RooDataHist object. - */ +/// Create a RooDataHist with the name `name` over the variables `vars` from the "contents" of `n`, and its "errors" +/// if it has any. std::unique_ptr RooJSONFactoryWSTool::readBinnedData(const JSONNode &n, const std::string &name, RooArgSet const &vars) { @@ -3212,10 +2608,7 @@ void RooJSONFactoryWSTool::exportModelConfig(RooFit::ModelConfig const &mc, int founddata = 0; if (simpdf) { // One analysis per combined data set with the same columns. - std::vector columns; - for (RooCategory const *column : selectionColumns(simpdf->indexCat())) { - columns.push_back(column->GetName()); - } + const std::vector columns = namesOf(selectionColumns(simpdf->indexCat())); for (auto const &data : combDataSets) { if (data.columns == columns) { exportSingleModelConfig(mc, std::string(mc.GetName()) + "_" + data.name, &data.name); @@ -3344,12 +2737,6 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc likelihood.terms.push_back({{pdf->GetName()}, 0.0}); } - auto addAux = [&](std::string const &name) { - auto &aux = likelihood.auxDistributions; - if (std::find_if(aux.begin(), aux.end(), [&](auto const &ref) { return ref.name == name; }) == aux.end()) { - aux.push_back({name}); - } - }; // The additional terms of the likelihood: the external constraints of the ModelConfig, and the constraint terms // of the channels of the model, which can be the components of a combined distribution (constraintsOfChannels()). // They are listed in "aux_distributions" with the observed values of their observables as constant parameters, or @@ -3375,14 +2762,9 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc } } std::vector constraintTerms; - auto addConstraintTerm = [&](std::string const &term) { - if (std::find(constraintTerms.begin(), constraintTerms.end(), term) == constraintTerms.end()) { - constraintTerms.push_back(term); - } - }; if (mc.GetExternalConstraints()) { for (const auto &constr : *mc.GetExternalConstraints()) { - addConstraintTerm(constr->GetName()); + addUnique(constraintTerms, constr->GetName()); } } // The constraint terms that the HistFactory exporter wrote for the channel `channel`, in the order of the document: @@ -3390,19 +2772,14 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc // names. auto writtenByHistFactory = [&](std::string const &channel) { std::vector out; - auto add = [&](std::string const &name) { - if (std::find(out.begin(), out.end(), name) == out.end()) { - out.push_back(name); - } - }; if (auto found = _export->constraintTermsOfPdf.find(channel); found != _export->constraintTermsOfPdf.end()) { for (std::string const &term : found->second) { - add(term); + addUnique(out, term); } } if (auto found = doc.rooFit.channelConstraints.find(channel); found != doc.rooFit.channelConstraints.end()) { for (auto const &ref : found->second) { - add(ref.name); + addUnique(out, ref.name); } } return out; @@ -3413,7 +2790,7 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc // constraint terms of the HistFactory channels are listed. for (std::string const &channel : channels) { for (std::string const &term : writtenByHistFactory(channel)) { - addConstraintTerm(term); + addUnique(constraintTerms, term); } } } else { @@ -3436,16 +2813,13 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc term = found->second; } } - auto &terms = termsOfChannel[item.channel]; - if (std::find(terms.begin(), terms.end(), term) == terms.end()) { - terms.push_back(term); - } + addUnique(termsOfChannel[item.channel], term); } for (std::string const &channel : channels) { std::vector const &found = termsOfChannel[channel]; if (!isHistFactoryChannel(channel)) { for (std::string const &term : found) { - addConstraintTerm(term); + addUnique(constraintTerms, term); } continue; } @@ -3467,7 +2841,7 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc "': the channel writes " + join(writtenSet) + ", but RooFit finds " + join(foundSet)); } for (std::string const &term : written) { - addConstraintTerm(term); + addUnique(constraintTerms, term); } } } @@ -3475,16 +2849,13 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc if (config().exportConstraintData && exportConstraintData(term, mc.GetGlobalObservables())) { likelihood.terms.push_back({{term}, HS3::DataRef{term + "_data"}}); } else { - addAux(term); + likelihood.auxDistributions.push_back({term}); } } doc.likelihoods.push_back(std::move(likelihood)); if (RooArgSet const *pois = mc.GetParametersOfInterest()) { - std::vector names; - names.reserve(pois->size()); - for (RooAbsArg const *arg : *pois) - names.push_back(arg->GetName()); + std::vector names = namesOf(*pois); std::sort(names.begin(), names.end()); for (auto const &name : names) { analysis.parametersOfInterest.push_back({name}); @@ -3523,10 +2894,7 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc info.mcName = mc.GetName(); // Without a data set, the document doesn't say what the observables are. if (!dataName && mc.GetObservables() && !mc.GetObservables()->empty()) { - std::vector names; - for (RooAbsArg const *arg : *mc.GetObservables()) { - names.push_back(arg->GetName()); - } + std::vector names = namesOf(*mc.GetObservables()); std::sort(names.begin(), names.end()); info.observables = std::move(names); } @@ -3595,7 +2963,7 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) // categorical axes, unless its observables are renamed per key (Config::renameCombinedObservables). if (selectionColumns(*d->get()).empty() || (!config().renameCombinedObservables && - !findMatchingSimultaneous(_workspace, selectionColumnNames(*d->get())))) { + !findMatchingSimultaneous(_workspace, namesOf(selectionColumns(*d->get()))))) { singleData.push_back(d); this->exportData(*d); } else { @@ -3654,6 +3022,33 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) } } + // The exporters have written the generic distributions to the JSON tree. Now that the tree has all of them, the + // document refers to their nodes, like a document that was read. + if (JSONNode const *distributions = n.find("distributions")) { + std::map nodes; + for (JSONNode const &node : distributions->children()) { + nodes.emplace(name(node), &node); + } + for (HS3::Distribution &dist : doc.distributions) { + auto found = nodes.find(dist.name); + if (auto *generic = std::get_if(&dist.body); generic && found != nodes.end()) { + generic->node = found->second; + } + } + } + + // A RooFit model often has a constraint both as a factor of the product of a channel and as a term of the + // likelihood (e.g. a constraint of a HistFactory modifier, an external constraint of the ModelConfig, or any + // constraint of a channel, see constraintsOfChannels()); RooFit counts it once. In HS3 it would enter twice. + for (auto const &[product, factors] : HS3::writeLikelihoodTermsOnce(doc, n)) { + for (std::string const &factor : factors) { + oocoutI(nullptr, IO) << "RooFitHS3: '" << factor << "' is a factor of '" << product + << "' and a term of its likelihood; it is written once, as a term of the likelihood" + << std::endl; + } + } + declareBookkeepingColumns(doc, _workspace); + // The document that RooFit writes has to pass the same checks as the ones that it reads. Until RooFit writes // documents that follow the current standard, problems are only reported. HS3::Diagnostics diagnostics; @@ -3668,12 +3063,9 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) HS3::report(diagnostics); HS3::writeDocument(n, doc); - // Each term of a likelihood once, and distinct variates (see sharedVariates()). Otherwise, the document is only - // written if explicitly allowed, e.g. to rename the variates in the JSON by hand. This is checked on the written - // document, which has the generic distributions. - writeLikelihoodTermsOnce(n); - declareBookkeepingColumns(n, _workspace); - if (std::vector const shared = sharedVariatesOfDocument(n); !shared.empty()) { + // Without distinct variates (see HS3::sharedVariates()), the document is only written if explicitly allowed, e.g. + // to rename the variates in the JSON by hand. + if (std::vector const shared = HS3::sharedVariates(doc); !shared.empty()) { if (!config().allowExportSharedVariates) { std::string text; for (std::string const &message : shared) { @@ -3694,23 +3086,14 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) } } -/** - * @brief Import the workspace from a JSON string. - * - * @param s The JSON string containing the workspace data. - * @return bool Returns true on successful import, false otherwise. - */ +/// Import an HS3 document that is given as a JSON string into the workspace. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::importJSONfromString(const std::string &s) { std::stringstream ss(s); return importJSON(ss); } -/** - * @brief Export the workspace to a JSON string. - * - * @return std::string The JSON string representing the exported workspace. - */ +/// Export the workspace as an HS3 document, and return it as a JSON string. std::string RooJSONFactoryWSTool::exportJSONtoString() { std::stringstream ss; @@ -3718,32 +3101,7 @@ std::string RooJSONFactoryWSTool::exportJSONtoString() return ss.str(); } -/** - * @brief Create a new JSON tree with version information. - * - * @return std::unique_ptr A unique pointer to the created JSON tree. - */ -std::unique_ptr RooJSONFactoryWSTool::createNewJSONTree() -{ - std::unique_ptr tree = JSONTree::create(); - JSONNode &n = tree->rootnode(); - n.set_map(); - auto &metadata = n["metadata"].set_map(); - - // add the mandatory hs3 version number - metadata["hs3_version"] << hs3VersionTag; - - appendNamedChild(metadata["packages"], "ROOT")["version"] << rootVersion(); - - return tree; -} - -/** - * @brief Export the workspace to JSON format and write to the output stream. - * - * @param os The output stream to write the JSON data to. - * @return bool Returns true on successful export, false otherwise. - */ +/// Export the workspace as an HS3 document, and write it as JSON to the stream. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::exportJSON(std::ostream &os) { std::unique_ptr tree = JSONTree::create(); @@ -3753,12 +3111,7 @@ bool RooJSONFactoryWSTool::exportJSON(std::ostream &os) return true; } -/** - * @brief Export the workspace to JSON format and write to the specified file. - * - * @param filename The name of the JSON file to create and write the data to. - * @return bool Returns true on successful export, false otherwise. - */ +/// Export the workspace as an HS3 document, and write it to a JSON file. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::exportJSON(std::string const &filename) { std::ofstream out(filename.c_str()); @@ -3816,7 +3169,7 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) std::optional validated; if (!HS3::hasErrors(diagnostics)) { HS3::ValidateOptions validateOptions; - validateOptions.strict = !isHS3VersionBefore(parsed.metadata.hs3Version, 0, 3); + validateOptions.strict = !(parsed.metadata.version < HS3::Version{0, 3}); validateOptions.isExternal = [this](std::string const &name) { return _workspace.obj(name) != nullptr; }; validated = HS3::validate(std::move(parsed), diagnostics, validateOptions); } @@ -3826,15 +3179,9 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) // The state of the import refers to the document, so it must not survive it, not even if there is an exception. struct ImportScope { RooJSONFactoryWSTool &tool; - ~ImportScope() - { - tool._import.reset(); - tool._hs3VersionInput.clear(); - } + ~ImportScope() { tool._import.reset(); } } importScope{*this}; - _hs3VersionInput = doc.metadata.hs3Version; - _import = std::make_unique(doc); createVariables(_workspace, _import->document.domains, _import->ranges); @@ -3913,10 +3260,10 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) // Now, read in analyses and likelihoods if there are any - // Distinct variates (see sharedVariates()). Documents of versions before 0.3, which RooFit wrote with shared + // Distinct variates (see HS3::sharedVariates()). Documents of versions before 0.3, which RooFit wrote with shared // variates itself, are read with a warning. std::set bookkeepingColumns; - for (auto const &[dataset, columns] : bookkeepingColumnsOfDocument(n)) { + for (auto const &[dataset, columns] : HS3::bookkeepingColumns(doc)) { for (std::string const &column : columns) { warning("RooFitHS3: the axis '" + column + "' of the components of the combined data set '" + dataset + "' is declared as a bookkeeping column without statistical meaning (misc.ROOT_internal." @@ -3924,8 +3271,8 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) bookkeepingColumns.insert(column); } } - for (std::string const &message : sharedVariatesOfDocument(n)) { - if (!isHS3VersionBefore(_hs3VersionInput, 0, 3)) { + for (std::string const &message : HS3::sharedVariates(doc)) { + if (!(doc.metadata.version < HS3::Version{0, 3})) { error("invalid HS3: " + message + ". The terms of a likelihood and the components of a combined distribution or data set must have " "distinct variates."); @@ -3936,24 +3283,7 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) } for (auto const &analysis : doc.analyses) { - // The variates of the distributions of the likelihood, see observablesWithoutData(). - std::set variates; - for (auto const &likelihood : doc.likelihoods) { - if (likelihood.name == analysis.likelihood.name) { - for (auto const &term : likelihood.terms) { - std::set const v = variatesOf(n, term.distribution.name); - variates.insert(v.begin(), v.end()); - // The columns of a combined distribution are observables as well. - JSONNode const *dist = findInSection(n, "distributions", term.distribution.name); - if (dist && typeOf(*dist) == "combined_dist") { - if (JSONNode const *columns = dist->find("columns")) { - addNames(*columns, variates); - } - } - } - } - } - importAnalysis(doc, analysis, _workspace, datasets, bookkeepingColumns, variates); + importAnalysis(doc, analysis, _workspace, datasets, bookkeepingColumns); } combineDatasets(doc, datasets); @@ -3977,29 +3307,17 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) importParameterStepWidths(_workspace, doc); } -/** - * @brief Imports a JSON file from the given input stream to the workspace. - * - * @param is The input stream containing the JSON data. - * @return bool Returns true on successful import, false otherwise. - */ +/// Import an HS3 document that is read as JSON from the stream into the workspace. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::importJSON(std::istream &is) { - // import a JSON file to the workspace std::unique_ptr tree = JSONTree::create(is); this->importAllNodes(tree->rootnode()); return true; } -/** - * @brief Imports a JSON file from the given filename to the workspace. - * - * @param filename The name of the JSON file to import. - * @return bool Returns true on successful import, false otherwise. - */ +/// Import the HS3 document of a JSON file into the workspace. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::importJSON(std::string const &filename) { - // import a JSON file to the workspace std::ifstream infile(filename.c_str()); if (!infile.is_open()) RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() invalid input file '" + filename + "'."); @@ -4044,8 +3362,7 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: this->importVariableElement(n); } else if (isData) { HS3::Diagnostics diagnostics; - HS3::Data data; - readValue(HS3::Cursor{n, diagnostics}, data); + HS3::Data data = HS3::parseData(n, diagnostics); HS3::report(diagnostics); _workspace.import(*loadData(data, _workspace, nullptr), RooFit::Silence()); } else { @@ -4088,35 +3405,21 @@ void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) } } -/** - * @brief Writes a warning message to the RooFit message service. - * - * @param str The warning message to be logged. - * @return std::ostream& A reference to the output stream. - */ +/// Write a warning to the RooFit message service, and return its stream. std::ostream &RooJSONFactoryWSTool::warning(std::string const &str) { return RooMsgService::instance().log(nullptr, RooFit::MsgLevel::WARNING, RooFit::ObjectHandling) << str << std::endl; } -/** - * @brief Writes an error message to the RooFit message service and throws a runtime_error. - * - * @param s The error message to be logged and thrown. - * @return void - */ +/// Write an error to the RooFit message service, and throw it as a std::runtime_error. void RooJSONFactoryWSTool::error(const char *s) { RooMsgService::instance().log(nullptr, RooFit::MsgLevel::ERROR, RooFit::IO) << s << std::endl; throw std::runtime_error(s); } -/** - * @brief Cleans up names to the HS3 standard - * - * @param str The string to be sanitized. - * @return std::string - */ +/// Replace some characters that are common in the names of RooFit objects, but that the HS3 standard doesn't allow in +/// names, if config().allowSanitizeNames is set. std::string RooJSONFactoryWSTool::sanitizeName(const std::string str) { std::string result; @@ -4202,36 +3505,22 @@ RooJSONFactoryWSTool::Config &RooJSONFactoryWSTool::config() // Sanitize all names in the workspace to be HS3 compliant RooWorkspace RooJSONFactoryWSTool::sanitizeWS(const RooWorkspace &ws) { - // Variables - RooWorkspace tmpWS = cleanWS(ws, false); - auto sanitizeIfNeeded = [](auto const &list) { - for (auto *obj : list) { - if (!isValidName(obj->GetName())) { - obj->SetName(sanitizeName(obj->GetName()).c_str()); - } - } - }; - sanitizeIfNeeded(tmpWS.allVars()); - sanitizeIfNeeded(tmpWS.allFunctions()); - sanitizeIfNeeded(tmpWS.allPdfs()); - sanitizeIfNeeded(tmpWS.allResolutionModels()); - // Datasets - for (auto *data : tmpWS.allData()) { - // Sanitize dataset name - if (!isValidName(data->GetName())) { - data->SetName(sanitizeName(data->GetName()).c_str()); - } - for (auto *obj : *data->get()) { + // Valid names stay as they are. + auto sanitize = [](auto const &objects) { + for (auto *obj : objects) { obj->SetName(sanitizeName(obj->GetName()).c_str()); } - } - for (auto *data : tmpWS.allEmbeddedData()) { - // Sanitize dataset name - data->SetName(sanitizeName(data->GetName()).c_str()); - for (auto *obj : *data->get()) { - obj->SetName(sanitizeName(obj->GetName()).c_str()); + }; + sanitize(tmpWS.allVars()); + sanitize(tmpWS.allFunctions()); + sanitize(tmpWS.allPdfs()); + sanitize(tmpWS.allResolutionModels()); + for (auto const &datasets : {tmpWS.allData(), tmpWS.allEmbeddedData()}) { + sanitize(datasets); + for (RooAbsData *data : datasets) { + sanitize(*data->get()); } } for (auto *snshObj : tmpWS.getSnapshots()) { @@ -4241,62 +3530,24 @@ RooWorkspace RooJSONFactoryWSTool::sanitizeWS(const RooWorkspace &ws) std::cerr << "Warning: found snapshot that is not a RooArgSet, skipping\n"; continue; } - - // Sanitize snapshot name - if (!isValidName(snsh->GetName())) { - snsh->setName(sanitizeName(snsh->GetName()).c_str()); - } - - // Sanitize the variables inside the snapshot - for (auto *arg : *snsh) { - if (!isValidName(arg->GetName())) { - arg->SetName(sanitizeName(arg->GetName()).c_str()); - } - } + snsh->setName(sanitizeName(snsh->GetName()).c_str()); + sanitize(*snsh); } - - // Generic objects (ModelConfigs, attributes, etc.) + // Generic objects, like the ModelConfigs with their sets for (auto *obj : tmpWS.allGenericObjects()) { - if (!isValidName(obj->GetName())) { - if (auto *named = dynamic_cast(obj)) { - named->SetName(sanitizeName(named->GetName()).c_str()); - } else { - std::cerr << "Warning: object " << obj->GetName() << " is not TNamed, cannot rename.\n"; - } + if (auto *named = dynamic_cast(obj)) { + named->SetName(sanitizeName(named->GetName()).c_str()); + } else if (!isValidName(obj->GetName())) { + std::cerr << "Warning: object " << obj->GetName() << " is not TNamed, cannot rename.\n"; } - if (auto *mc = dynamic_cast(obj)) { - // Sanitize ModelConfig name - if (!isValidName(mc->GetName())) { - mc->SetName(sanitizeName(mc->GetName()).c_str()); - } - - // Sanitize the sets inside ModelConfig - for (auto *obs : mc->GetObservables()->get()) { - if (obs) { - obs->SetName(sanitizeName(obs->GetName()).c_str()); - } - } - for (auto *poi : mc->GetParametersOfInterest()->get()) { - if (poi) { - poi->SetName(sanitizeName(poi->GetName()).c_str()); - } - } - for (auto *nuis : mc->GetNuisanceParameters()->get()) { - if (nuis) { - nuis->SetName(sanitizeName(nuis->GetName()).c_str()); - } - } - for (auto *glob : mc->GetGlobalObservables()->get()) { - if (glob) { - glob->SetName(sanitizeName(glob->GetName()).c_str()); - } - } + sanitize(*mc->GetObservables()); + sanitize(*mc->GetParametersOfInterest()); + sanitize(*mc->GetNuisanceParameters()); + sanitize(*mc->GetGlobalObservables()); } } - std::string wsName = std::string{ws.GetName()} + "_sanitized"; RooWorkspace newWS = cleanWS(tmpWS, false); - newWS.SetName(wsName.c_str()); - + newWS.SetName((std::string{ws.GetName()} + "_sanitized").c_str()); return newWS; } From e2ce3ed99e1362a072f2874e09194ca58cb64ba4 Mon Sep 17 00:00:00 2001 From: Carsten Burgard Date: Thu, 19 Nov 2026 17:54:39 +0100 Subject: [PATCH 5/9] [RF][HS3] Follow the HS3 0.3 specification fixes (#24201) Bring the HS3 import and export in line with the fixes that went into the standard before v0.3 (HS3 PR 154, with the decisions of the HS3 issues 156 and 157): https://github.com/hep-statistics-serialization-standard/hep-statistics-serialization-standard/pull/154 Where 0.3 forbids or changes something that older documents rely on, the new behaviour applies to documents with hs3_version 0.3 or later. Older documents are read as before, usually with a warning. HS3 0.3: * Data axes: an infinite bound of a continuous axis is left out instead of written as null or as the largest double, and an axis with one bound sets only that bound on import. The validator reports binned axes with infinite bounds, data axes with a fixed "value", and data axes whose range differs from the domain of the same variate. * Point data is read and written as a RooDataSet with one entry of one variable. Without axes, and for a number given in-line in "data", the variable is the single variate of the paired distribution. * A ModelConfig without data is written as a likelihood with "data": null. The old "data": [0] is read as "no data" only in documents before 0.3, and is an observation of 0 otherwise. * landau_dist: the location is written as "mu"; "mean" is still read. * mixture_dist: "extended" is required with one coefficient per summand. An explicit false is now honoured, with a RooAddPdf of the normalized coefficients that can't be extended. * Expressions: TRUE, FALSE, === and !== are translated for TFormula. The validator reports objects with a reserved name (PI, EULER, TRUE, FALSE, ifelse, functions). A generic expression that refers to a distribution is rejected on import, and the export fails for what it would have to write as one, e.g. a RooFormulaVar that depends on a pdf. * HistFactory: the "constraints" of a modifier add no term. Every named constraint must contribute exactly once to each likelihood with the channel, counted by the entries of the data it is paired with. * HistFactory: staterror gammas with named constraints keep the range and constness of the document, and are not configured from the errors of the samples. Auxiliary measurements (constraint terms), independent of HS3 0.3: * A RooLognormal in RooFit's default parametrization (median, shape) can be a component of an array_dist, so config().exportArrayDist and config().exportConstraintData also apply to such constraints. * The helper functions of literal constants get valid identifiers, "const_1p2_lognormal_log" instead of "1.2_lognormal_log", with all digits of the value. The old names are still read. * Objects that the import creates for something RooFit has no class for, like the RooProdPdf that holds an array_dist together, carry the attribute "hs3_helper" and are only written if something refers to them. Written as a product_dist, that product kept the constraints out of arrays in every later export. * With exportConstraintData, an array of constraints of a RooSimultaneous is a component of the combined_dist, with its data set in the combined data set, instead of a separate likelihood term. A channel that consists of constraint terms only can now be written. These components are marked in misc.ROOT_internal.auxiliary_channels and read back as constraint terms, or kept as channels with the new, EXPERIMENTAL config().importConstraintsAsChannels. * On import, a constraint term with a global observable becomes a factor of the channels that use its parameters again, instead of an external constraint of the ModelConfig. A channel that is no product is wrapped in a RooProdPdf "_with_constraints". The pdf of the model stays the sampling distribution of the auxiliary measurements in a round trip, which Asimov data and toys rely on. * Which of x and mean of a Gaussian constraint is the observable is determined before any pdf is written, so that it can't disagree with the axis of the constraint's data set. Documents before 0.3 that are published on HEPData: * Two histosys modifiers of the same name in a sample, as ROOT 6.37 wrote them, are a warning instead of an error, and are read as one parameter that interpolates the sample once per modifier. * The HistFactory export writes the formulas of a channel as custom modifiers if splitting them into one modifier per factor makes the channel invalid, e.g. two normfactors of the same name for "b*b". Each change has a test in testRooFitHS3. The new testHS3AuxiliaryMeasurements writes a three-channel HistFactory workspace in all four layouts of exportArrayDist x exportConstraintData and compares the documents, workspaces, likelihood scans and toys. The likelihoods of rf515_hfJSON.json and of the hand-written 0.3 documents of the tests now list their constraints in "aux_distributions". Assisted-by: claude-code:claude-opus-5-5 (cherry picked from commit 2bacb4b33e86c0a5b8b1985fe72a31272400a792) --- .../hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h | 39 +- roofit/hs3/src/HS3Builder.h | 3 + roofit/hs3/src/HS3Document.h | 32 +- roofit/hs3/src/HS3Parser.cxx | 11 + roofit/hs3/src/HS3Validate.cxx | 160 +- roofit/hs3/src/HS3Variates.cxx | 172 ++ roofit/hs3/src/HS3Writer.cxx | 46 +- roofit/hs3/src/JSONFactories_HistFactory.cxx | 78 +- roofit/hs3/src/JSONFactories_RooFitCore.cxx | 210 ++- roofit/hs3/src/JSONIOUtils.cxx | 85 + roofit/hs3/src/JSONIOUtils.h | 56 +- roofit/hs3/src/RooJSONFactoryWSTool.cxx | 882 ++++++++++- roofit/hs3/test/CMakeLists.txt | 1 + .../hs3/test/testHS3AuxiliaryMeasurements.cxx | 1391 +++++++++++++++++ roofit/hs3/test/testRooFitHS3.cxx | 1390 +++++++++++++++- tutorials/roofit/roofit/rf515_hfJSON.json | 7 + 16 files changed, 4413 insertions(+), 150 deletions(-) create mode 100644 roofit/hs3/test/testHS3AuxiliaryMeasurements.cxx diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 4af617545fbe3..fc860142eeb02 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -76,12 +76,24 @@ class RooJSONFactoryWSTool { /// the JSON. bool allowExportSharedVariates = false; // Write array distributions (HS3 `array_dist`). If false, no `array_dist` is written: HistFactory constraints - // are named individually, and a RooProdPdf tagged as array distribution is written as `product_dist`. + // are named individually, and a RooProdPdf tagged as array distribution is written as `product_dist`. The + // RooProdPdf that RooFit created itself when it read an `array_dist` is only written if another object uses + // it; otherwise its components are written on their own. bool exportArrayDist = true; // How the constraint terms of HistFactory models enter the likelihood: listed in "aux_distributions", with - // the observed values as constant parameters (false), or listed in "distributions", each paired with a data set - // holding the observed values of its observables (true). + // the observed values as constant parameters (false), or paired with a data set holding the observed values + // of their observables (true). An array of constraints of a model that is a RooSimultaneous is then a component + // of the combined distribution, and its data set a component of the combined data set, so that the combined + // distribution is the sampling distribution of all measurements. Other constraints are terms of the likelihood. bool exportConstraintData = false; + /// EXPERIMENTAL: how auxiliary measurements that are components of a combined distribution (a constraint term + /// paired with the data set of its observed values, as RooFit writes them with `exportConstraintData`) are + /// read. If false, they become constraint terms of the model as in HistFactory, with global observables set + /// to the observed values; RooFit recognizes the components that it wrote like this by a mark in the "misc" + /// section of the document. If true, or without that mark, they stay components of the RooSimultaneous, with + /// their data set of one entry in the combined data set, so that the RooSimultaneous is the sampling + /// distribution of all measurements. This option and the form of the resulting workspace may still change. + bool importConstraintsAsChannels = false; }; static Config &config(); @@ -250,6 +262,21 @@ class RooJSONFactoryWSTool { /// version that can't be parsed. RooFit::HS3::Version hs3Version() const; + /// Whether the component `component` of the combined distribution `combinedDist` of the document that is + /// currently being imported is read as a constraint term instead (see Config::importConstraintsAsChannels). + bool isAuxiliaryChannelReadAsConstraint(std::string const &combinedDist, std::string const &component) const; + + /// Whether the constraint term `pdf` contributes to the likelihoods of the document that is currently being + /// imported as a component of a combined distribution with data, by itself or as a component of an array + /// distribution. It is then a measurement of its own, and must not be multiplied into the distributions that + /// name it as a constraint as well. + bool isConstraintReadAsChannel(RooAbsPdf const &pdf) const; + + /// The other terms of the likelihoods of the document that is currently being imported that have the distribution + /// `dist` as a term: their other distributions and their auxiliary distributions. These are the candidates for + /// constraint terms of `dist`. + std::vector otherLikelihoodTerms(std::string const &dist) const; + private: template T *requestImpl(const std::string &objname); @@ -269,7 +296,7 @@ class RooJSONFactoryWSTool { } } void exportArrayDists(); - bool exportConstraintData(std::string const &term, RooArgSet const *globalObservables); + bool exportConstraintData(std::string const &term, RooArgSet const *globalObservables, std::string const &dataName); void exportSimultaneous(RooSimultaneous const &simPdf); void recordCategory(RooAbsCategory const &cat); @@ -299,6 +326,10 @@ class RooJSONFactoryWSTool { std::vector channelConstraints); // member variables + /// The components of combined distributions that are read as constraint terms, by combined distribution. + std::map> _auxiliaryChannelsAsConstraints; + /// The components of combined distributions that have data, and are read as such. + std::set _componentsWithData; RooWorkspace &_workspace; // objects to represent intermediate information diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h index 718f5cc46c6e5..5402a01c9e406 100644 --- a/roofit/hs3/src/HS3Builder.h +++ b/roofit/hs3/src/HS3Builder.h @@ -45,6 +45,9 @@ RooArgSet createObservables(std::vector const &axes); BinnedAxis toBinnedAxis(RooRealVar const &var); BinnedAxis toBinnedAxis(std::string const &name, RooAbsBinning const &binning); +/// The axis of unbinned or point data for a variable. Infinite bounds are left out: the axis is unbounded there. +ContinuousAxis toContinuousAxis(RooRealVar const &var); + /// The name of an object, or the value of a literal constant. NameOrNumber toNameOrNumber(RooAbsArg const &arg); diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index b2c27d4353605..3cfce71cd57c5 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -110,6 +110,9 @@ struct ContinuousAxis { std::string name; std::optional min; std::optional max; + /// Whether the axis has a fixed "value", which is only allowed in domains. Older versions of RooFit wrote it for + /// the constant observables of data sets. The value is not used. + bool hasFixedValue = false; }; /// The axis of binned data. @@ -207,15 +210,19 @@ struct Data { std::variant body; }; +/// The data of a likelihood term: a data set, a number given in-line, or none (std::monostate) in a pure model. struct LikelihoodTerm { DistributionRef distribution; - std::variant data; + std::variant data; }; struct Likelihood { std::string name; std::vector terms; std::vector auxDistributions; + /// Whether "data" is null: the likelihood only groups the distributions of a model without data (e.g. for toy + /// generation), and it can't be evaluated before a data set is created. + bool pureModel = false; }; struct Analysis { @@ -488,6 +495,9 @@ struct RooFitInternal { /// The constraint terms of HistFactory distributions that no modifier names, by distribution. They are /// also listed in the "aux_distributions" of the likelihoods, which is what other implementations use. std::map> channelConstraints; + /// The components of a combined distribution that are auxiliary measurements (constraint terms paired with the + /// data set of their observed values), by combined distribution. RooFit can read them back as constraint terms. + std::map> auxiliaryChannels; /// Axes that the components of a combined data set share, but that are no variate of any distribution, e.g. an /// event number, by combined data set. They have no statistical meaning, and may be shared. std::map> bookkeepingColumns; @@ -657,10 +667,30 @@ void factorOutDefaultInterpolation(HistFactoryDist &dist); /// by the components of a combined data set, except for bookkeeping columns. std::vector sharedVariates(HS3Document const &doc); +/// In HS3 0.3 (HS3 PR #154, decision on HS3 #156), the "constraints" of a HistFactory modifier add no term to a +/// likelihood: every likelihood that contains the channel must list each named distribution exactly once, in +/// "distributions" or "aux_distributions", or as a factor of a product_dist or a component of a combined_dist reached +/// from them. One message for each named distribution of each likelihood that doesn't. For older documents, RooFit +/// adds the named constraints itself (see sharedVariates()). +/// +/// A distribution contributes one factor per entry of the data that it is paired with, so that being listed is not +/// the same as contributing: a component of a combined_dist without a component of the paired combined data set, or +/// with one that has no entries, contributes nothing. A constraint that contributes once elsewhere is then fine, but +/// since listing it in the model and again on its own is probably unintended, `warnings` gets a message. So it does +/// for every component of a combined distribution that has no data component at all. +std::vector unlistedConstraints(HS3Document const &doc, std::vector *warnings = nullptr); + +/// The distributions that are components of a combined distribution of a likelihood, with a component of the paired +/// combined data set that has entries: measurements of their own. +std::set componentsWithData(HS3Document const &doc); + /// The variates of the "distributions" of a likelihood, which it pairs with data, with the columns of its combined /// distributions. std::set variatesOf(HS3Document const &doc, Likelihood const &likelihood); +/// The variates of the distribution `distribution`, without the columns of a combined distribution. +std::set variatesOf(HS3Document const &doc, std::string const &distribution); + /// The axes that several components of a combined data set have, by combined data set. std::map> sharedAxes(HS3Document const &doc); diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 7452ed3b35206..001a6a3c4746b 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -323,6 +323,8 @@ void describe(Reader &ar, DomainAxis &axis) void describe(Reader &ar, ContinuousAxis &axis) { ar("name", axis.name); + // A fixed axis is only allowed in domains (HS3 0.3), which the validation reports. + axis.hasFixedValue = ar.has("value"); readBound(ar, "min", axis.min, -infinity); readBound(ar, "max", axis.max, infinity); } @@ -618,6 +620,14 @@ void describe(Reader &ar, Likelihood &likelihood) return; } Cursor data = ar.child("data"); + // A pure model, i.e. distributions without any data, which a ModelConfig can refer to. + if (data.node().is_null()) { + likelihood.pureModel = true; + for (auto &distribution : distributions) { + likelihood.terms.push_back({std::move(distribution), std::monostate{}}); + } + return; + } if (!data.node().is_seq()) { data.error("must be an array"); return; @@ -1130,6 +1140,7 @@ void readMisc(Cursor const &root, RooFitInternal &out) readMap(ar, "combined_datasets", out.combinedDatasets); readMap(ar, "ModelConfigs", out.modelConfigs); readMap(ar, "histfactory_constraints", out.channelConstraints); + readMap(ar, "auxiliary_channels", out.auxiliaryChannels); readMap(ar, "bookkeeping_columns", out.bookkeepingColumns); } diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index 133defab4c520..96186d6db0a99 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -57,6 +57,12 @@ class Validator { requireDistribution(ref.name, "misc.ROOT_internal.combined_distributions." + item.first, "distribution"); } } + for (auto const &item : _doc.rooFit.auxiliaryChannels) { + requireDistribution(item.first, "misc.ROOT_internal.auxiliary_channels", "distribution"); + for (auto const &ref : item.second) { + requireDistribution(ref.name, "misc.ROOT_internal.auxiliary_channels." + item.first, "component"); + } + } for (auto const &item : _doc.rooFit.channelConstraints) { requireDistribution(item.first, "misc.ROOT_internal.histfactory_constraints", "distribution"); for (auto const &ref : item.second) { @@ -130,6 +136,7 @@ class Validator { void checkCategoricalAxes(); void checkLikelihoods(); void checkAnalyses(); + void checkNormalizationRanges(Analysis const &analysis, std::string const &path); void checkData(); void checkCombinedData(); @@ -140,6 +147,15 @@ class Validator { std::set _parameters; }; +// The names that the HS3 expression language reserves for itself: its literals and functions (HS3 0.3, "Identifiers"). +static bool isReservedName(std::string const &name) +{ + static const std::set reserved{"PI", "EULER", "TRUE", "FALSE", "ifelse", "exp", "log", + "sqrt", "abs", "floor", "ceil", "pow", "min", "max", + "sin", "cos", "tan", "asin", "acos", "atan", "erf"}; + return reserved.count(name) > 0; +} + // HS3 requires the names of all objects to be unique within a document, including the parameters. RooFit writes such // documents, e.g. for a range and a pdf of one name, so only ambiguous clashes in current documents are errors. void Validator::collectNames() @@ -220,6 +236,22 @@ void Validator::collectNames() } } } + // Objects that expressions can refer to must not carry the reserved names of the expression language. + auto checkNotReserved = [&](char const *what, std::string const &name) { + if (isReservedName(name)) { + strictError("", "the " + std::string(what) + " '" + name + + "' has a name that the HS3 expression language reserves for itself"); + } + }; + for (auto const &dist : _doc.distributions) { + checkNotReserved("distribution", dist.name); + } + for (auto const &func : _doc.functions) { + checkNotReserved("function", func.name); + } + for (std::string const ¶meter : _parameters) { + checkNotReserved("parameter", parameter); + } for (std::string const ¶meter : _parameters) { if (char const *section = sectionOf(parameter)) { strictError("", "the name '" + parameter + "' is used in \"" + section + @@ -448,11 +480,102 @@ void Validator::checkAnalyses() if (analysis.prior && !isDefinedIn(analysis.prior->name, "distributions") && !isExternal(analysis.prior->name)) { strictError(path + ".prior", "the distribution '" + analysis.prior->name + "' does not exist"); } + checkNormalizationRanges(analysis, path); + } +} + +// A variate is normalized over the axis of the paired data set, or else over its axis in the domains of the analysis. +// If both exist, they must give the same range, where an absent bound is equal only to an absent bound (HS3 0.3). +void Validator::checkNormalizationRanges(Analysis const &analysis, std::string const &path) +{ + using Range = std::pair, std::optional>; + auto bound = [](std::optional value) { return value && !std::isinf(*value) ? value : std::nullopt; }; + auto format = [](Range const &range) { + auto one = [](std::optional value) { return value ? std::to_string(*value) : std::string("none"); }; + return "[" + one(range.first) + ", " + one(range.second) + "]"; + }; + + std::map domainRanges; + for (auto const &ref : analysis.domains) { + for (auto const &domain : _doc.domains) { + if (domain.name != ref.name) { + continue; + } + for (auto const &axisVariant : domain.axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis || axis->fixedValue) { + continue; // a fixed axis gives no range + } + std::optional min = axis->min; + std::optional max = axis->max; + auto const *edges = axis->binning ? std::get_if(&*axis->binning) : nullptr; + if (edges && !edges->edges.empty()) { + min = min ? min : edges->edges.front(); + max = max ? max : edges->edges.back(); + } + domainRanges[axis->name] = {bound(min), bound(max)}; + } + } + } + if (domainRanges.empty()) { + return; + } + + auto check = [&](std::string const &dataName, std::string const &axisName, Range const &dataRange) { + auto found = domainRanges.find(axisName); + if (found != domainRanges.end() && found->second != dataRange) { + strictError(path, "the axis '" + axisName + "' of the data set '" + dataName + "' has the range " + + format(dataRange) + ", but the domains of the analysis '" + analysis.name + + "' give it the range " + format(found->second)); + } + }; + std::function checkData = [&](std::string const &dataName) { + for (auto const &data : _doc.data) { + if (data.name != dataName) { + continue; + } + auto checkAxes = [&](auto const &axes) { + for (auto const &axisVariant : axes) { + std::visit( + overloaded{[&](ContinuousAxis const &a) { check(data.name, a.name, {bound(a.min), bound(a.max)}); }, + [&](BinnedAxis const &a) { check(data.name, a.name, {bound(a.min), bound(a.max)}); }, + [](CategoricalAxis const &) {}}, + axisVariant); + } + }; + std::visit(overloaded{[&](PointData const &d) { + for (auto const &a : d.axes) { + check(data.name, a.name, {bound(a.min), bound(a.max)}); + } + }, + [&](UnbinnedData const &d) { checkAxes(d.axes); }, + [&](BinnedData const &d) { checkAxes(d.axes); }, + [&](CombinedData const &d) { + for (auto const &component : d.components) { + checkData(component.data.name); + } + }}, + data.body); + } + }; + for (auto const &likelihood : _doc.likelihoods) { + if (likelihood.name != analysis.likelihood.name) { + continue; + } + for (auto const &term : likelihood.terms) { + if (auto const *data = std::get_if(&term.data)) { + checkData(data->name); + } + } } } std::size_t Validator::checkAxis(BinnedAxis const &axis, std::string const &path) { + // Unlike continuous axes, binned axes always have finite bounds. + if (std::isinf(axis.min) || std::isinf(axis.max)) { + error(path, "the binned axis '" + axis.name + "' needs finite bounds"); + } if (auto const *regular = std::get_if(&axis.binning)) { if (regular->nbins <= 0) { error(path, "the axis '" + axis.name + "' needs a positive number of bins"); @@ -497,11 +620,29 @@ void Validator::checkData() { for (std::size_t i = 0; i < _doc.data.size(); ++i) { auto const &data = _doc.data[i]; + const std::string path = at("data", i, data.name); + // A fixed axis is only allowed in domains (HS3 0.3). Older versions of RooFit wrote the constant observables of + // data sets like this. + auto checkNotFixed = [&](ContinuousAxis const &axis) { + if (axis.hasFixedValue) { + strictError(path, "the axis '" + axis.name + "' of a data set must not have a fixed \"value\""); + } + }; + if (auto const *point = std::get_if(&data.body)) { + for (auto const &axis : point->axes) { + checkNotFixed(axis); + } + } else if (auto const *unbinned = std::get_if(&data.body)) { + for (auto const &axis : unbinned->axes) { + if (auto const *continuous = std::get_if(&axis)) { + checkNotFixed(*continuous); + } + } + } auto const *binned = std::get_if(&data.body); if (!binned) { continue; } - const std::string path = at("data", i, data.name); std::optional nBinsOrNone = numBins(binned->axes, path); if (!nBinsOrNone) { continue; @@ -628,12 +769,19 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const error(samplePath, "the name of the sample is not unique in the channel"); } checkSize(samplePath + ".data.contents", sample.contents.size(), "\"contents\""); - // These modifiers create objects that are named after them. + // These modifiers create objects that are named after them. Older versions of RooFit wrote a parameter that + // interpolates a sample more than once as several histosys modifiers of the same name, which are still read. std::set objectNames; - auto checkUniqueName = [&](std::string const &where, char const *type, std::string const &modifierName) { + auto checkUniqueName = [&](std::string const &where, char const *type, std::string const &modifierName, + bool allowedBeforeHS3v03 = false) { if (!objectNames.insert(std::string(type) + " " + modifierName).second) { - error(where, - std::string("there is another ") + type + " modifier named '" + modifierName + "' in the sample"); + const std::string message = + std::string("there is another ") + type + " modifier named '" + modifierName + "' in the sample"; + if (allowedBeforeHS3v03) { + strictError(where, message); + } else { + error(where, message); + } } }; if (sample.errors && sample.errors->size() != sample.contents.size()) { @@ -666,7 +814,7 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const checkInterpolation(m.interpolation, "normsys", m.name); }, [&](HistoSys const &m) { - checkUniqueName(modifierPath, "histosys", m.name); + checkUniqueName(modifierPath, "histosys", m.name, true); checkSingleParameter(m); checkInterpolation(m.interpolation, "histosys", m.name); checkSize(modifierPath + ".data.hi.contents", m.hi.contents.size(), "\"contents\""); diff --git a/roofit/hs3/src/HS3Variates.cxx b/roofit/hs3/src/HS3Variates.cxx index 9af3005cf6a7a..10750c240ab74 100644 --- a/roofit/hs3/src/HS3Variates.cxx +++ b/roofit/hs3/src/HS3Variates.cxx @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -310,6 +311,35 @@ PartVariates componentAxes(HS3Document const &doc, CombinedData const &combined) return parts; } +// The data set `name` of the document, or nullptr. +Data const *findData(HS3Document const &doc, std::string const &name) +{ + auto found = std::find_if(doc.data.begin(), doc.data.end(), [&](Data const &d) { return d.name == name; }); + return found != doc.data.end() ? &*found : nullptr; +} + +// The number of entries of the data set `name` of the document, i.e. the number of factors that a distribution +// paired with it contributes to a likelihood: one for point data and binned data, the number of entries for unbinned +// data. Without such a data set, the document is not valid, which is reported elsewhere: one is assumed. +int numDataEntries(HS3Document const &doc, std::string const &name) +{ + Data const *data = findData(doc, name); + auto const *unbinned = data ? std::get_if(&data->body) : nullptr; + return unbinned ? static_cast(unbinned->numEntries()) : 1; +} + +// The labels of the key of a component of a combined distribution, like the parser reads those of combined data. +std::vector keyOf(JSONNode const &component) +{ + std::vector key; + if (JSONNode const *node = component.find("key"); node && node->is_seq()) { + for (JSONNode const &label : node->children()) { + key.push_back(label.is_container() ? std::string{} : label.val()); + } + } + return key; +} + } // namespace std::set variatesOf(HS3Document const &doc, Likelihood const &likelihood) @@ -328,6 +358,14 @@ std::set variatesOf(HS3Document const &doc, Likelihood const &likel return variates; } +std::set variatesOf(HS3Document const &doc, std::string const &distribution) +{ + std::set variates; + std::set visited; + Distributions{doc}.addVariates(distribution, variates, visited); + return variates; +} + std::map> sharedAxes(HS3Document const &doc) { std::map> out; @@ -368,6 +406,140 @@ std::map> bookkeepingColumns(HS3Document cons return out; } +std::vector unlistedConstraints(HS3Document const &doc, std::vector *warnings) +{ + Distributions distributions{doc}; + std::vector messages; + for (Likelihood const &likelihood : doc.likelihoods) { + std::map contributions; + // The combined distributions in which a distribution is a component without data. + std::map> withoutData; + std::set named; + // `combined` is the combined distribution whose component without data the distribution is reached through. + std::function contribute = + [&](std::string const &name, int entries, std::string const &combined) { + contributions[name] += entries; + if (entries == 0) { + withoutData[name].insert(combined); + } + if (auto const *histFactory = distributions.find(name)) { + for (std::string const &constraint : namedConstraints(*histFactory)) { + named.insert(constraint); + } + } else if (JSONNode const *generic = distributions.nodeOf(name, "histfactory_dist")) { + for (std::string const &constraint : namedConstraints(*generic)) { + named.insert(constraint); + } + } else if (JSONNode const *product = distributions.nodeOf(name, "product_dist")) { + for (std::string const &factor : namesIn(*product, "factors")) { + contribute(factor, entries, combined); + } + } + }; + // The components of a combined distribution, each with the entries of the component of the paired combined data + // set that has the same key. Without paired data (a pure model), every component contributes once. + auto contributeCombined = [&](std::string const &distName, JSONNode const &dist, Data const *data) { + auto const *combinedData = data ? std::get_if(&data->body) : nullptr; + JSONNode const *components = dist.find("components"); + if (!components) { + return; + } + for (JSONNode const &component : components->children()) { + JSONNode const *name = component.find("dist"); + if (!name || !isName(*name)) { + continue; + } + int entries = 1; + if (combinedData) { + entries = 0; + bool found = false; + std::vector const key = keyOf(component); + for (CombinedDataComponent const &dataComponent : combinedData->components) { + if (dataComponent.key == key) { + found = true; + entries = numDataEntries(doc, dataComponent.data.name); + } + } + if (!found && warnings) { + warnings->push_back("the component '" + name->val() + "' of the combined distribution '" + distName + + "' has no component in the data set '" + data->name + "' that the likelihood '" + + likelihood.name + "' pairs it with, so it does not contribute to the likelihood"); + } + } + contribute(name->val(), entries, distName); + } + }; + for (LikelihoodTerm const &term : likelihood.terms) { + // The paired data: a data set, a value given in-line, or nothing (a pure model). + auto const *dataRef = std::get_if(&term.data); + Data const *data = dataRef ? findData(doc, dataRef->name) : nullptr; + std::string const &name = term.distribution.name; + if (JSONNode const *combined = distributions.nodeOf(name, "combined_dist")) { + ++contributions[name]; + contributeCombined(name, *combined, data); + } else { + contribute(name, data ? numDataEntries(doc, data->name) : 1, ""); + } + } + for (DistributionRef const &aux : likelihood.auxDistributions) { + contribute(aux.name, 1, ""); + } + for (std::string const &constraint : named) { + const int n = contributions[constraint]; + auto const ®ions = withoutData[constraint]; + std::string inRegions; + for (std::string const ®ion : regions) { + inRegions += (inRegions.empty() ? "'" : ", '") + region + "'"; + } + if (n != 1) { + messages.push_back("the constraint '" + constraint + + "' named by a modifier of a HistFactory channel contributes " + std::to_string(n) + + " times to the likelihood '" + likelihood.name + + "', but it must contribute exactly once" + + (n == 0 && !regions.empty() ? " (it is a component of the combined distribution " + + inRegions + " without data, which is no observation)" + : "")); + } else if (!regions.empty() && warnings) { + warnings->push_back("the constraint '" + constraint + "' is a term of the likelihood '" + likelihood.name + + "' and also a component of the combined distribution " + inRegions + + " without data. It contributes once, as the term"); + } + } + } + return messages; +} + +std::set componentsWithData(HS3Document const &doc) +{ + Distributions distributions{doc}; + std::set out; + for (Likelihood const &likelihood : doc.likelihoods) { + for (LikelihoodTerm const &term : likelihood.terms) { + auto const *dataRef = std::get_if(&term.data); + Data const *data = dataRef ? findData(doc, dataRef->name) : nullptr; + auto const *combinedData = data ? std::get_if(&data->body) : nullptr; + JSONNode const *dist = distributions.nodeOf(term.distribution.name, "combined_dist"); + JSONNode const *components = dist ? dist->find("components") : nullptr; + if (!components || !combinedData) { + continue; + } + for (JSONNode const &component : components->children()) { + JSONNode const *name = component.find("dist"); + if (!name || !isName(*name)) { + continue; + } + std::vector const key = keyOf(component); + for (CombinedDataComponent const &dataComponent : combinedData->components) { + if (dataComponent.key == key && numDataEntries(doc, dataComponent.data.name) > 0) { + out.insert(name->val()); + } + } + } + } + } + return out; +} + std::vector sharedVariates(HS3Document const &doc) { Distributions distributions{doc}; diff --git a/roofit/hs3/src/HS3Writer.cxx b/roofit/hs3/src/HS3Writer.cxx index b8027150f7c50..41e7962e86ecd 100644 --- a/roofit/hs3/src/HS3Writer.cxx +++ b/roofit/hs3/src/HS3Writer.cxx @@ -27,16 +27,6 @@ namespace RooFit { namespace HS3 { namespace { -// Infinite bounds are written as null. -void writeBound(JSONNode &node, double value) -{ - if (std::isinf(value)) { - node.set_null(); - } else { - node << value; - } -} - template void writeRefs(JSONNode &node, std::vector> const &refs) { @@ -62,15 +52,17 @@ void writeBinning(JSONNode &node, Binning const &binning) } } +// An infinite or missing bound of a continuous data axis is omitted, which means that the range is unbounded. JSON null +// is not defined for bounds. void writeAxis(JSONNode &node, ContinuousAxis const &axis) { node.set_map(); node["name"] << axis.name; - if (axis.min) { - writeBound(node["min"], *axis.min); + if (axis.min && !std::isinf(*axis.min)) { + node["min"] << *axis.min; } - if (axis.max) { - writeBound(node["max"], *axis.max); + if (axis.max && !std::isinf(*axis.max)) { + node["max"] << *axis.max; } } @@ -276,12 +268,20 @@ void writeLikelihood(JSONNode &likelihoods, Likelihood const &likelihood) { JSONNode &node = appendNamed(likelihoods, likelihood.name); JSONNode &distributions = node["distributions"].set_seq(); - JSONNode &data = node["data"].set_seq(); - for (auto const &term : likelihood.terms) { - distributions.append_child() << term.distribution.name; - std::visit(overloaded{[&](DataRef const &ref) { data.append_child() << ref.name; }, - [&](double value) { writeNumber(data.append_child(), value); }}, - term.data); + if (likelihood.pureModel) { + for (auto const &term : likelihood.terms) { + distributions.append_child() << term.distribution.name; + } + node["data"].set_null(); + } else { + JSONNode &data = node["data"].set_seq(); + for (auto const &term : likelihood.terms) { + distributions.append_child() << term.distribution.name; + std::visit(overloaded{[&](DataRef const &ref) { data.append_child() << ref.name; }, + [&](double value) { writeNumber(data.append_child(), value); }, + [&](std::monostate) { data.append_child().set_null(); }}, + term.data); + } } if (!likelihood.auxDistributions.empty()) { writeRefs(node["aux_distributions"], likelihood.auxDistributions); @@ -365,6 +365,12 @@ void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) writeRefs(constraints[item.first], item.second); } } + if (!internal.auxiliaryChannels.empty()) { + JSONNode &channels = rootInternal()["auxiliary_channels"].set_map(); + for (auto const &item : internal.auxiliaryChannels) { + writeRefs(channels[item.first], item.second); + } + } if (!internal.bookkeepingColumns.empty()) { JSONNode &columns = rootInternal()["bookkeeping_columns"].set_map(); for (auto const &item : internal.bookkeepingColumns) { diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 932f5d2d6e1fe..2067e2081da17 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -404,7 +404,8 @@ void configureGammasWithConstraints(RooArgList const &gammas, std::vector const &parnames, std::vector const &vals, HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, - double gammaMax, double minSigma, std::size_t nBins, std::string const &context) + double gammaMax, double minSigma, std::size_t nBins, std::string const &context, + bool configureGammas = true) { RooWorkspace &ws = *tool.workspace(); @@ -453,7 +454,9 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, } constraintPdfs.push_back(pdf); } - configureGammasWithConstraints(gammas, vals, constraintPdfs, minSigma); + // Without `configureGammas`, nothing is derived from `vals`: the gammas keep their range and constness, and + // only get the initial error that their constraint distribution encodes. + configureGammasWithConstraints(gammas, configureGammas ? vals : std::vector{}, constraintPdfs, minSigma); } if (vals.empty()) { @@ -548,6 +551,7 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi hf.SetTitle(sampleName.c_str()); SampleElements elems; + std::map histoSysCount; elems.shapeElems.add(tool.wsEmplace(newName(prefixedName + "_binWidth"), hf, true)); const bool hasStatError = std::any_of(sample.modifiers.begin(), sample.modifiers.end(), @@ -598,8 +602,12 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi [&](HS3::HistoSys const &mod) { RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); elems.histNps.add(param); + // Documents before HS3 0.3 can have several histosys modifiers of the same name in a sample (see the + // validation). Their histograms are told apart by a counter. + const int count = ++histoSysCount[mod.name]; + const std::string objName = count == 1 ? mod.name : mod.name + "_" + std::to_string(count); auto addVariation = [&](RooArgList &list, HS3::HistoSysVariation const &variation, const char *suffix) { - const std::string name = newName(mod.name + suffix + prefixedName); + const std::string name = newName(objName + suffix + prefixedName); list.add( tool.wsEmplace(name, ctx.observables, HS3::createDataHist(name, ctx.observables, variation.contents, @@ -986,6 +994,8 @@ std::vector contents(RooDataHist const &dataHist) struct ExportedChannel { std::string name; HS3::HistFactoryDist dist; + /// Whether a formula among the factors of a sample was written as several modifiers, see readChannel(). + bool decomposedFormulas = false; // The variables of the first histogram, named after the observables of the channel. std::unique_ptr observables; std::size_t nBins = 0; @@ -1096,8 +1106,10 @@ bool isShapeModifierFunction(ParamHistFunc const &phf, ExportedChannel const &ch return true; } +// Read the channel from the RooRealSumPdf. With `decomposeFormulas`, a RooFormulaVar among the factors of a sample +// that is a product is written as one modifier per factor; otherwise, like any other function, as one custom modifier. ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf, - std::vector const &channelConstraints) + std::vector const &channelConstraints, bool decomposeFormulas = true) { ExportedChannel channel; @@ -1231,7 +1243,11 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna expression.ReplaceAll(("x[" + std::to_string(i) + "]").c_str(), p->GetName()); expression.ReplaceAll(("@" + std::to_string(i)).c_str(), p->GetName()); } - auto components = splitTopLevelProduct(expression.Data()); + std::vector components; + if (decomposeFormulas) { + components = splitTopLevelProduct(expression.Data()); + } + channel.decomposedFormulas |= !components.empty(); if (components.empty()) { // it's not a product, let's just treat it as an unknown element custom.push_back({formula->GetName()}); @@ -1391,6 +1407,23 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna return channel; } +// Whether the distribution of the channel passes the checks of a HistFactory distribution, with its duplicate +// modifiers combined like for the export. +bool isValidChannel(ExportedChannel const &channel, std::string const &pdfname) +{ + if (!channel.observables) { + return true; // nothing was read that could be checked + } + HS3::HistFactoryDist dist = channel.dist; + HS3::Diagnostics diagnostics; + HS3::mergeDuplicateModifiers(dist, channel.name, diagnostics); + for (auto *obs : static_range_cast(*channel.observables)) { + dist.axes.push_back(HS3::toBinnedAxis(*obs)); + } + HS3::validateHistFactoryDist(dist, pdfname, diagnostics); + return !HS3::hasErrors(diagnostics); +} + // The statistical uncertainties of the samples are reconstructed from the constraints of the gamma parameters, with // the (arbitrary) convention that all samples have the same relative uncertainty. void configureStatError(ExportedChannel &channel) @@ -1446,7 +1479,7 @@ bool usedOnlyByHistFactoryChannels(RooJSONFactoryWSTool &tool, RooAbsPdf const & } } for (RooAbsArg const *client : constraint.clients()) { - if (tool.workspace()->arg(client->GetName()) != client) { + if (tool.workspace()->arg(client->GetName()) != client || isSkippedHelper(*client, *tool.workspace())) { continue; // not part of what is exported } if (!client->getAttribute(arrayDistTag) && !channelSum(*client)) { @@ -1554,6 +1587,15 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } ExportedChannel channel = readChannel(tool, pdfname, sumpdf, constraints); + // Writing a formula as several modifiers must not be what makes the channel invalid, e.g. because a parameter that + // occurs twice in the formula becomes two normfactors of the same name. If it is, the formulas are written as + // custom modifiers instead. + if (channel.decomposedFormulas && !isValidChannel(channel, pdfname)) { + ExportedChannel plain = readChannel(tool, pdfname, sumpdf, constraints, /*decomposeFormulas=*/false); + if (isValidChannel(plain, pdfname)) { + channel = std::move(plain); + } + } HS3::HistFactoryDist &dist = channel.dist; // sanity checks @@ -1696,10 +1738,14 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c errs[i] = sumW[i] > 0. ? std::sqrt(sumW2[i]) / sumW[i] : 0.; } - mcStatObject = - &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, - statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, - dist.statError.relThreshold, errs.size(), "the staterror modifier of channel '" + name + "'"); + // In HS3 0.3 (HS3 PR #154, decision on HS3 #156), the named constraints of a staterror are authoritative: the + // range, the constness and the step width of the gammas come from the document, and nothing is derived from + // the errors of the samples. Older documents relied on RooFit configuring the gammas from them. + const bool configureGammas = tool.hs3Version() < HS3::Version{0, 3}; + mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, + statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, + dist.statError.relThreshold, errs.size(), + "the staterror modifier of channel '" + name + "'", configureGammas); } ChannelContext ctx{tool, dist, observables, name, channelName, mcStatObject, constraints}; @@ -1719,6 +1765,18 @@ void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string c } } + // A constraint that is a measurement of its own, as a component of a combined distribution with data, contributes + // there, and is not multiplied into the channel. + { + RooArgSet ownMeasurements; + for (auto *constraint : static_range_cast(constraints)) { + if (tool.isConstraintReadAsChannel(*constraint)) { + ownMeasurements.add(*constraint); + } + } + constraints.remove(ownMeasurements); + } + // With constraints, the channel is the product of the sum of the samples and the constraints. auto &sum = tool.wsEmplace(constraints.empty() ? name : channelName + "_model", funcs, coefs, true); sum.SetTitle(name.c_str()); diff --git a/roofit/hs3/src/JSONFactories_RooFitCore.cxx b/roofit/hs3/src/JSONFactories_RooFitCore.cxx index 5a3b2382ca4d4..61d94ff8e6b95 100644 --- a/roofit/hs3/src/JSONFactories_RooFitCore.cxx +++ b/roofit/hs3/src/JSONFactories_RooFitCore.cxx @@ -271,6 +271,11 @@ void translateImportedExpression(TString &expr) { replaceIdentifier(expr, "PI", "TMath::Pi()"); replaceIdentifier(expr, "EULER", "TMath::E()"); + // The boolean literals, and the exact comparisons, which are the ones of TFormula. + replaceIdentifier(expr, "TRUE", "1"); + replaceIdentifier(expr, "FALSE", "0"); + expr.ReplaceAll("!==", "!="); + expr.ReplaceAll("===", "=="); expr = ifElseToTernary(expr.Data()).c_str(); } @@ -318,7 +323,13 @@ bool importFormulaArg(RooJSONFactoryWSTool *tool, const JSONNode &p) translateImportedExpression(formula); RooArgList dependents; for (const auto &d : extractArguments(formula.Data())) { - dependents.add(*tool->request(d, name)); + RooAbsReal *dependent = tool->request(d, name); + // HS3 0.3 (HS3 PR #154) only allows expressions to refer to parameters, variables and functions. + if (dynamic_cast(dependent) && !(tool->hs3Version() < HS3::Version{0, 3})) { + RooJSONFactoryWSTool::error("the expression of '" + name + "' refers to the distribution '" + d + + "', which is not allowed in HS3"); + } + dependents.add(*dependent); } RooArg_t arg{name.c_str(), formula, dependents}; importFormulaBinnings(arg, p); @@ -340,6 +351,41 @@ bool importProdPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) return true; } +// RooLognormal has two parametrizations. In the standard one, its parameters are the mean "mu" and the width "sigma" of +// log(x), like in the lognormal_dist of HS3. In the other one, which is the default of RooFit, they are the median m0 +// of x and the shape parameter k, with mu = log(m0) and sigma = log(k). Such a distribution is written with "mu" and +// "sigma" naming the functions log(m0) and log(k), which are named after m0 and k with the suffix below. The suffix +// is also how the parametrization is recognized when reading: the distribution is then created from m0 and k +// themselves instead of the functions. The two functions below are the only places that know this convention, for +// distributions on their own and for the components of array distributions alike. +constexpr const char *lognormalLogSuffix = "_lognormal_log"; + +// What a RooLognormal that is not in the standard parametrization writes as "mu" (for its median) or "sigma" (for its +// shape parameter): the name of the function log(`param`). The function is written as well if there is a `tool`; +// without one, only the name is given. +std::string lognormalLogName(RooAbsReal const ¶m, RooJSONFactoryWSTool *tool) +{ + if (tool) { + return tool->exportTransformed(¶m, lognormalLogSuffix, "log(%s)"); + } + return transformedBaseName(param) + lognormalLogSuffix; +} + +// The parameters to create a RooLognormal from, for the "mu" and "sigma" of a lognormal_dist. +struct LognormalParameters { + std::string median; + std::string shape; + bool standardParametrization = true; +}; + +LognormalParameters lognormalParameters(std::string const &mu, std::string const &sigma) +{ + if (!endsWith(mu, lognormalLogSuffix)) { + return {mu, sigma, true}; + } + return {transformedOriginal(mu, lognormalLogSuffix), transformedOriginal(sigma, lognormalLogSuffix), false}; +} + // The name of the pdf that represents the component of an array distribution over the observable `observable`. std::string arrayComponentName(std::string const &observable) { @@ -473,6 +519,8 @@ bool importArrayDist(RooJSONFactoryWSTool *tool, const JSONNode &p) RooProdPdf array{name.c_str(), name.c_str(), pdfs}; array.setAttribute(arrayDistTag); + // The product only exists to hold the components together. + array.setAttribute(helperObjectTag); tool->wsImport(array); return true; } @@ -486,8 +534,10 @@ struct ArrayComponent { // The type, observable and fields of `pdf` as a component of an array distribution, with the fields of the standalone // exporters, or std::nullopt if `pdf` cannot be a component. A Gaussian whose mean is the recorded `observable` is -// written with x and mean swapped. -std::optional arrayComponentOf(RooAbsPdf const &pdf, std::string const *observable = nullptr) +// written with x and mean swapped. The functions that a field names are only written if there is a `tool` (see +// lognormalLogName()); without one, this only describes the component. +std::optional +arrayComponentOf(RooAbsPdf const &pdf, std::string const *observable = nullptr, RooJSONFactoryWSTool *tool = nullptr) { ArrayComponent out; RooAbsReal const *x = nullptr; @@ -506,13 +556,15 @@ std::optional arrayComponentOf(RooAbsPdf const &pdf, std::string } out.fields = {{"mean", HS3::toNameOrNumber(*mean)}, {"sigma", HS3::toNameOrNumber(gauss->getSigma())}}; } else if (auto const *lognormal = dynamic_cast(&pdf)) { - if (!lognormal->useStandardParametrization()) { - return std::nullopt; - } out.type = "lognormal_dist"; x = &lognormal->getX(); - out.fields = {{"mu", HS3::toNameOrNumber(lognormal->getMedian())}, - {"sigma", HS3::toNameOrNumber(lognormal->getShapeK())}}; + if (lognormal->useStandardParametrization()) { + out.fields = {{"mu", HS3::toNameOrNumber(lognormal->getMedian())}, + {"sigma", HS3::toNameOrNumber(lognormal->getShapeK())}}; + } else { + out.fields = {{"mu", lognormalLogName(lognormal->getMedian(), tool)}, + {"sigma", lognormalLogName(lognormal->getShapeK(), tool)}}; + } } else { return std::nullopt; } @@ -553,6 +605,10 @@ bool importCombinedDist(RooJSONFactoryWSTool *tool, const JSONNode &p) if (!component.is_map() || !component.has_child("dist")) { RooJSONFactoryWSTool::error("every component of " + context + " must have a \"key\" and a \"dist\""); } + // An auxiliary measurement that is read as a constraint term of the model instead. + if (tool->isAuxiliaryChannelReadAsConstraint(name, component["dist"].val())) { + continue; + } std::vector const key = readSelectionKey(component, columns, context); std::string const label = selectionLabel(key); if (pdfMap.find(label) != pdfMap.end()) { @@ -562,6 +618,28 @@ bool importCombinedDist(RooJSONFactoryWSTool *tool, const JSONNode &p) pdfMap[label] = tool->request(component["dist"].val(), name); } + // In RooFit, the constraint terms of a channel are factors of it. A channel that is a product gets them when the + // likelihood is read; the other channels are wrapped in a product here, since the components of a RooSimultaneous + // can't be replaced later. The candidates are the other terms of the likelihoods that have this distribution. + { + std::vector terms; + for (std::string const &term : tool->otherLikelihoodTerms(name)) { + if (auto *pdf = tool->tryRequest(term)) { + terms.push_back(pdf); + } + } + RooArgSet channels; + for (auto const &item : pdfMap) { + channels.add(*item.second, /*silent=*/true); + } + for (auto &item : pdfMap) { + if (!dynamic_cast(item.second)) { + item.second = + &channelWithConstraints(*tool, *item.second, constraintsOfChannel(*item.second, terms, channels)); + } + } + } + if (columns.size() == 1) { tool->wsImport(RooSimultaneous{name.c_str(), name.c_str(), pdfMap, *columns.front()}); return true; @@ -595,12 +673,41 @@ bool importAddPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) RooJSONFactoryWSTool::error("mixture_dist '" + name + "' is declared as not extended, but it has " + counts + ", which defines an extended mixture of extended distributions"); } - if (!extended && nCoefficients == nSummands) { - RooJSONFactoryWSTool::warning("RooFitHS3: mixture_dist '" + name + "' is declared as not extended, but " + - "RooFit can use it as extended because it has one coefficient per summand. " + - "Don't use the extended likelihood term when fitting it."); + if (!extended && nCoefficients == nSummands && nSummands > 0) { + // A RooAddPdf with one coefficient per summand can always be extended. The non-extended mixture is the one + // with the normalized coefficients c_i / sum(c), of which the last one follows from the others. + RooArgList summands = tool->requestArgList(p, "summands"); + RooArgList coefficients = tool->requestArgList(p, "coefficients"); + std::string sum; + for (std::size_t i = 0; i < nCoefficients; ++i) { + sum += (i == 0 ? "x[" : "+x[") + std::to_string(i) + "]"; + } + RooArgList fractions; + for (std::size_t i = 0; i + 1 < nCoefficients; ++i) { + fractions.add(tool->wsEmplace(name + "_fraction_" + std::to_string(i), + ("x[" + std::to_string(i) + "]/(" + sum + ")").c_str(), + coefficients)); + } + tool->wsEmplace(name, summands, fractions); + return true; + } + } else { + const std::size_t nSummands = p.has_child("summands") ? p["summands"].num_children() : 0; + const std::size_t nCoefficients = p.has_child("coefficients") ? p["coefficients"].num_children() : 0; + // HS3 0.3 requires the flag. Where it decides about the Poisson term, a document without it is invalid. + if (nCoefficients == nSummands && nSummands > 0 && !(tool->hs3Version() < HS3::Version{0, 3})) { + RooJSONFactoryWSTool::error("mixture_dist '" + name + + "' has one coefficient per summand, but no \"extended\" flag, which HS3 " + "requires to decide whether the mixture is extended"); } } + if (p.has_child("summands") && p["summands"].num_children() > 0 && + (!p.has_child("coefficients") || p["coefficients"].num_children() == 0) && + !(tool->hs3Version() < HS3::Version{0, 3})) { + RooJSONFactoryWSTool::warning("RooFitHS3: mixture_dist '" + name + + "' has no coefficients, which HS3 0.3 doesn't define. It is read as the sum of " + "its extended summands."); + } if (!tool->requestArgList(p, "coefficients").empty()) { tool->wsEmplace(name, tool->requestArgList(p, "summands"), tool->requestArgList(p, "coefficients")); @@ -815,6 +922,23 @@ bool importExtendPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) return true; } +// HS3 0.3 calls the location parameter of the Landau distribution "mu", since it is neither the mean nor the most +// probable value. Documents before 0.3 used "mean", which is still read. +bool importLandau(RooJSONFactoryWSTool *tool, const JSONNode &p) +{ + const std::string name(RooJSONFactoryWSTool::name(p)); + const bool legacyKey = !p.has_child("mu") && p.has_child("mean"); + if (legacyKey && !(tool->hs3Version() < HS3::Version{0, 3})) { + RooJSONFactoryWSTool::warning("RooFitHS3: the landau_dist '" + name + + "' uses the key \"mean\", which HS3 0.3 renamed to \"mu\"."); + } + RooAbsReal *x = tool->requestArg(p, "x"); + RooAbsReal *mu = tool->requestArg(p, legacyKey ? "mean" : "mu"); + RooAbsReal *sigma = tool->requestArg(p, "sigma"); + tool->wsEmplace(name, *x, *mu, *sigma); + return true; +} + bool importLogNormal(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); @@ -822,14 +946,11 @@ bool importLogNormal(RooJSONFactoryWSTool *tool, const JSONNode &p) // Same mechanism to undo the parameter transformation as in the // importExponential() function (see comments in that function for more info). - const std::string muName = p["mu"].val(); - const std::string sigmaName = p["sigma"].val(); - const bool isTransformed = endsWith(muName, "_lognormal_log"); - const std::string suffixToRemove = isTransformed ? "_lognormal_log" : ""; - RooAbsReal *mu = tool->request(removeSuffix(muName, suffixToRemove), name); - RooAbsReal *sigma = tool->request(removeSuffix(sigmaName, suffixToRemove), name); + const LognormalParameters params = lognormalParameters(p["mu"].val(), p["sigma"].val()); + RooAbsReal *median = tool->request(params.median, name); + RooAbsReal *shape = tool->request(params.shape, name); - tool->wsEmplace(name, *x, *mu, *sigma, !isTransformed); + tool->wsEmplace(name, *x, *median, *shape, params.standardParametrization); return true; } @@ -872,8 +993,8 @@ bool importExponential(RooJSONFactoryWSTool *tool, const JSONNode &p) const std::string constParamName = p["c"].val(); const bool isInverted = endsWith(constParamName, "_exponential_inverted"); - const std::string suffixToRemove = isInverted ? "_exponential_inverted" : ""; - RooAbsReal *c = tool->request(removeSuffix(constParamName, suffixToRemove), name); + RooAbsReal *c = tool->request( + isInverted ? transformedOriginal(constParamName, "_exponential_inverted") : constParamName, name); tool->wsEmplace(name, *x, *c, !isInverted); @@ -1064,16 +1185,6 @@ bool importGaussian(RooJSONFactoryWSTool *tool, const JSONNode &p) return true; } -bool importLandau(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *x = tool->requestArg(p, "x"); - RooAbsReal *mean = tool->requestArg(p, "mean"); - RooAbsReal *sigma = tool->requestArg(p, "sigma"); - tool->wsEmplace(name, *x, *mean, *sigma); - return true; -} - bool importPowerSum(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); @@ -1267,6 +1378,12 @@ void exportAddPdf(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) RooJSONFactoryWSTool::fillSeq(elem["summands"], pdf.pdfList()); RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); elem["extended"] << (pdf.extendMode() != RooArg_t::CanNotBeExtended); + // A RooAddModel is written as a mixture_model, which is not part of HS3. + if (pdf.coefList().empty() && std::is_same_v) { + RooJSONFactoryWSTool::warning("RooFitHS3: the RooAddPdf '" + std::string(pdf.GetName()) + + "' has no coefficients. It is written as a mixture_dist without coefficients, " + "which HS3 0.3 doesn't define, so other implementations might not read it."); + } } template @@ -1470,6 +1587,13 @@ bool ternariesToIfElse(std::string const &in, std::string &out) template void exportFormulaArg(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { + // HS3 0.3 (HS3 PR #154) only allows expressions to refer to parameters, variables and functions. + for (std::size_t i = 0; i < pdf.nParameters(); ++i) { + if (auto const *dependent = dynamic_cast(pdf.getParameter(i))) { + RooJSONFactoryWSTool::error(std::string("the expression of '") + pdf.GetName() + "' refers to the pdf '" + + dependent->GetName() + "', which HS3 doesn't allow"); + } + } TString expression(pdf.expression()); cleanExpression(expression); // If the tokens follow the "x[#]" convention, the square braces enclosing each number @@ -1585,8 +1709,8 @@ void exportLogNormal(RooJSONFactoryWSTool *tool, RooLognormal const &pdf, JSONNo elem["mu"] << m0.GetName(); elem["sigma"] << k.GetName(); } else { - elem["mu"] << tool->exportTransformed(&m0, "_lognormal_log", "log(%s)"); - elem["sigma"] << tool->exportTransformed(&k, "_lognormal_log", "log(%s)"); + elem["mu"] << lognormalLogName(m0, tool); + elem["sigma"] << lognormalLogName(k, tool); } } @@ -1610,6 +1734,15 @@ void exportMultiVarGaussian(RooJSONFactoryWSTool *, RooMultiVarGaussian const &p void exportTFnBinding(RooJSONFactoryWSTool *, RooTFnBinding const &pdf, JSONNode &elem) { + // HS3 0.3 (HS3 PR #154) only allows expressions to refer to parameters, variables and functions. + for (RooAbsCollection const *list : {&pdf.observables(), &pdf.parameters()}) { + for (RooAbsArg const *arg : *list) { + if (dynamic_cast(arg)) { + RooJSONFactoryWSTool::error(std::string("the expression of '") + pdf.GetName() + "' refers to the pdf '" + + arg->GetName() + "', which HS3 doesn't allow"); + } + } + } TString formula(pdf.function().GetExpFormula()); formula.ReplaceAll("x", pdf.observables()[0].GetName()); @@ -1760,7 +1893,7 @@ void exportGamma(RooJSONFactoryWSTool *, RooGamma const &pdf, JSONNode &elem) void exportLandau(RooJSONFactoryWSTool *, RooLandau const &pdf, JSONNode &elem) { writeReference(elem["x"], pdf.getX()); - writeReference(elem["mean"], pdf.getMean()); + writeReference(elem["mu"], pdf.getMean()); writeReference(elem["sigma"], pdf.getSigma()); } @@ -1963,7 +2096,8 @@ std::optional arrayComponentInfo(RooAbsPdf const &pdf, std:: return ArrayComponentInfo{component->type, component->x}; } -void writeArrayDist(JSONNode &elem, std::string const &name, std::vector const &components, +void writeArrayDist(RooJSONFactoryWSTool &tool, JSONNode &elem, std::string const &name, + std::vector const &components, std::map const &observableOf) { auto error = [&name](std::string const &message) { @@ -1976,11 +2110,11 @@ void writeArrayDist(JSONNode &elem, std::string const &name, std::vector observables; for (RooAbsPdf const *pdf : components) { auto recorded = observableOf.find(pdf->GetName()); - auto comp = arrayComponentOf(*pdf, recorded != observableOf.end() ? &recorded->second : nullptr); + auto comp = arrayComponentOf(*pdf, recorded != observableOf.end() ? &recorded->second : nullptr, &tool); if (!comp) { error("'" + std::string(pdf->GetName()) + - "' cannot be a component, only RooPoisson, RooGaussian and RooLognormal (standard parametrization) " - "over a named observable are supported"); + "' cannot be a component, only RooPoisson, RooGaussian and RooLognormal over a named observable " + "are supported"); } if (!comps.empty() && comp->type != comps.front().type) { error("the components '" + std::string(components.front()->GetName()) + "' (" + comps.front().type + diff --git a/roofit/hs3/src/JSONIOUtils.cxx b/roofit/hs3/src/JSONIOUtils.cxx index 70c13837ac21b..7c6d46e394645 100644 --- a/roofit/hs3/src/JSONIOUtils.cxx +++ b/roofit/hs3/src/JSONIOUtils.cxx @@ -11,6 +11,9 @@ #include #include +#include +#include +#include #include #include #include @@ -67,6 +70,88 @@ std::string removeSuffix(std::string_view str, std::string_view suffix) namespace { +constexpr std::string_view literalPrefix = "const_"; + +// The shortest decimal representation of a value that is read back as the same value. +std::string shortestNumber(double value) +{ + char buffer[32]; + for (int precision = 1; precision <= 17; ++precision) { + std::snprintf(buffer, sizeof(buffer), "%.*g", precision, value); + if (std::strtod(buffer, nullptr) == value) { + break; + } + } + return buffer; +} + +// The number that the identifier of a literal constant spells, or an empty string if `name` is no such identifier. +std::string numberOfLiteralIdentifier(std::string_view name) +{ + if (!startsWith(name, literalPrefix) || name.size() == literalPrefix.size()) { + return ""; + } + std::string number; + for (char c : name.substr(literalPrefix.size())) { + if (c == 'p') { + number += '.'; + } else if (c == 'm') { + number += '-'; + } else if (std::isdigit(static_cast(c)) || c == 'e') { + number += c; + } else { + return ""; + } + } + char *end = nullptr; + std::strtod(number.c_str(), &end); + return end == number.c_str() + number.size() ? number : ""; +} + +} // namespace + +std::string transformedArgument(RooAbsReal const ¶m) +{ + if (!isLiteralConstVar(param)) { + return param.GetName(); + } + // In parentheses if negative, such that "-%s" doesn't become "--1". + const std::string number = shortestNumber(param.getVal()); + return number[0] == '-' ? "(" + number + ")" : number; +} + +std::string transformedBaseName(RooAbsReal const ¶m) +{ + if (!isLiteralConstVar(param)) { + return param.GetName(); + } + std::string out{literalPrefix}; + for (char c : shortestNumber(param.getVal())) { + if (c == '.') { + out += 'p'; + } else if (c == '-') { + out += 'm'; + } else if (c != '+') { + out += c; + } + } + return out; +} + +bool isLiteralIdentifier(std::string_view name) +{ + return !numberOfLiteralIdentifier(name).empty(); +} + +std::string transformedOriginal(std::string_view name, std::string_view suffix) +{ + const std::string base = removeSuffix(name, suffix); + const std::string number = numberOfLiteralIdentifier(base); + return number.empty() ? base : number; +} + +namespace { + std::vector numBinsPerVariable(RooAbsCollection const &vars) { std::vector nBins; diff --git a/roofit/hs3/src/JSONIOUtils.h b/roofit/hs3/src/JSONIOUtils.h index d7b9b15a17ad6..ee445c6e5d265 100644 --- a/roofit/hs3/src/JSONIOUtils.h +++ b/roofit/hs3/src/JSONIOUtils.h @@ -12,6 +12,7 @@ #include "HS3Document.h" class RooAbsArg; +class RooAbsBinning; class RooAbsCategory; class RooAbsCollection; class RooAbsPdf; @@ -20,6 +21,7 @@ class RooArgList; class RooArgSet; class RooCategory; class RooDataHist; +class RooJSONFactoryWSTool; class RooWorkspace; class ParamHistFunc; @@ -28,12 +30,40 @@ class ParamHistFunc; // observable. inline constexpr const char *arrayDistTag = "hs3_array_dist"; +// An object with this attribute is a helper: RooFit created it to represent something of an HS3 document that it has +// no class for, e.g. the RooProdPdf that holds the components of an array distribution together, or the parts of a +// composite structure that stands for one HS3 object. A helper is not a statement of the user about the model, so: +// - it is only written if another object that is written refers to it by name, or if the HS3 object that it stands +// for is written (see isSkippedHelper()), but not just because it is in the workspace; +// - a helper that is not written is ignored wherever the export looks at the users of an object, e.g. to decide +// whether the object is a top-level object or whether a constraint can be a component of an array. +// The attribute is set by the importers and is not written to the document. Objects created by users don't have it +// and are always written. +inline constexpr const char *helperObjectTag = "hs3_helper"; + // True if `arg` is a RooConstVar whose name is a plain number (i.e. a literal constant). bool isLiteralConstVar(RooAbsArg const &arg); +// Whether `arg` is a helper (see helperObjectTag) that the export of the workspace `ws` does not write: nothing that +// is written refers to it, i.e. it has no clients in the workspace other than helpers that are skipped themselves and +// no ModelConfig names it, and the HS3 object that it stands for is not written either. Such an object is treated as +// if it were not in the workspace. +bool isSkippedHelper(RooAbsArg const &arg, RooWorkspace const &ws); + +// The constraint terms among `terms` that belong to the channel `channel` of a model with the channels `channels`: +// the ones with a global observable (a variable that no channel depends on) and with another variable that the +// channel depends on. An array of constraints is one term. +std::vector +constraintsOfChannel(RooAbsPdf const &channel, std::vector const &terms, RooArgSet const &channels); + +// The channel with its constraint terms as RooFit and HistFactory have them: a product of the constraints and the +// channel, named "_with_constraints", from which RooFit takes the constraints when it builds a likelihood. +// Returns the channel itself if it has none. +RooAbsPdf & +channelWithConstraints(RooJSONFactoryWSTool &tool, RooAbsPdf &channel, std::vector const &constraints); + // The HS3 type and the observable of `pdf` as a component of an array distribution (RooPoisson, RooGaussian or -// RooLognormal in the standard parametrization, over a named observable), or std::nullopt if `pdf` cannot be a -// component. +// RooLognormal, over a named observable), or std::nullopt if `pdf` cannot be a component. // If `observable` names the mean of a RooGaussian, the Gaussian is written with x and mean swapped (see // RooJSONFactoryWSTool::setConstraintObservable()). struct ArrayComponentInfo { @@ -43,8 +73,9 @@ struct ArrayComponentInfo { std::optional arrayComponentInfo(RooAbsPdf const &pdf, std::string const *observable = nullptr); // Write the array distribution `name` with the given components into `elem`. `observableOf` gives the observable -// recorded for a component by name, if any. -void writeArrayDist(RooFit::Detail::JSONNode &elem, std::string const &name, +// recorded for a component by name, if any. The `tool` writes the functions that the components need besides, e.g. +// for a RooLognormal that is not in the standard parametrization. +void writeArrayDist(RooJSONFactoryWSTool &tool, RooFit::Detail::JSONNode &elem, std::string const &name, std::vector const &components, std::map const &observableOf); @@ -59,6 +90,23 @@ bool startsWith(std::string_view str, std::string_view prefix); bool endsWith(std::string_view str, std::string_view suffix); std::string removeSuffix(std::string_view str, std::string_view suffix); +// Some RooFit classes are parametrized differently than their HS3 distribution, e.g. RooLognormal by default. They +// are written with a function of the parameter (RooJSONFactoryWSTool::exportTransformed()), which is named after the +// parameter with a suffix that says what the function is. A literal constant has no name that could be part of an +// identifier ("1.2", "-3", "1e-05"), so its value is spelled as one: "const_1p2", "const_m3", "const_1em05". +// +// What a transformed function is named after: the name of the parameter, or the identifier for the value of a +// literal constant. +std::string transformedBaseName(RooAbsReal const ¶m); +// What a transformed function is a function of: the name of the parameter, or the value of a literal constant with +// as many digits as it takes to read it back unchanged (in parentheses if it is negative). +std::string transformedArgument(RooAbsReal const ¶m); +// The inverse: what to create the RooFit object from, given the name of a transformed function with this suffix. This +// is the name of the parameter, or the value of a literal constant as a string. +std::string transformedOriginal(std::string_view name, std::string_view suffix); +// Whether a name is the identifier for the value of a literal constant. +bool isLiteralIdentifier(std::string_view name); + // A RooHistFunc or RooHistPdf is evaluated with its own observables (`funcObs`), which are mapped by position onto // the variables of its histogram (`histObs`) and need not have the same names, e.g. after a RooCustomizer replaced // them. The variables of the histogram `dh`, in the order of the histogram, each named after the observable that is diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 77a58119f13f0..0388c0c1b1fdf 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -172,6 +173,10 @@ struct RooFit::HS3::ToolExportState { /// The constraint terms of the exported HistFactory channels, as written, and the data sets written for them. std::map> constraintTermsOfPdf; std::set writtenConstraintData; + /// The constraint terms that are written as components of a combined distribution, by distribution, and the + /// labels that they add to its index category, by category. See exportSingleModelConfig(). + std::map> auxiliaryChannels; + std::map> auxiliaryLabels; /// The observable of a constraint term, by the name of the term, see setConstraintObservable(). std::map constraintObservables; /// Set by the exporter of an object that is interpreted by the document. @@ -543,6 +548,225 @@ void importParameterStepWidths(RooWorkspace &workspace, HS3Document const &doc) } } +// RooFit writes an array of constraints with the data set of its observed values as a component of the combined +// distribution of the model (see exportSingleModelConfig()), and marks it in misc.ROOT_internal.auxiliary_channels. +// Unless these components are to stay components (Config::importConstraintsAsChannels), the document is changed +// here to what RooFit wrote before: the component and its data set become a term of every likelihood that has the +// combined distribution, which is then read as a constraint term with global observables. The label of the component +// is taken out of the index category. Returns the components that are no longer components of their combined +// distribution, which is a generic object that can't be changed here: its importer leaves them out. +// +// The same is done for a document that lists a distribution twice: as a component of a combined distribution whose +// data set has no entries for it, and as a term of its own with data. It contributes once, as the term, but a +// component without data is not a constraint term for RooFit, which would leave it out of the likelihood. Such a +// document is read like the ones above, with a warning. If the components stay components (`asChannels`), the data +// of the term becomes the data of the component instead, and the term is removed; the components are returned in +// `componentsWithData`. +std::map> +readAuxiliaryChannelsAsConstraints(HS3Document &doc, bool asChannels, std::set &dataOfConstraints, + std::set &componentsWithData) +{ + std::map> dropped; + std::set labels; + // The label of a marked component, by combined distribution and component. + std::map> labelOf; + + auto isName = [](JSONNode const &node) { return !node.is_container() && !node.is_null() && !node.is_number(); }; + auto findCombinedDist = [&](std::string const &name) -> JSONNode const * { + for (HS3::Distribution const &dist : doc.distributions) { + auto const *generic = std::get_if(&dist.body); + if (generic && dist.name == name && generic->type == "combined_dist") { + return generic->node; + } + } + return nullptr; + }; + auto findData = [&](std::string const &name) -> HS3::Data * { + for (HS3::Data &data : doc.data) { + if (data.name == name) { + return &data; + } + } + return nullptr; + }; + // Components without data that are also terms with data. A data set can only be a component of one combined data + // set, so that the data of a term gets a copy for every further one; room for them is reserved, since the loop + // keeps pointers into the data sets. + doc.data.reserve(2 * doc.data.size() + 16); + std::set dataOfComponents; + for (HS3::Likelihood &likelihood : doc.likelihoods) { + std::set termsToRemove; + for (HS3::LikelihoodTerm const &term : likelihood.terms) { + JSONNode const *combinedDist = findCombinedDist(term.distribution.name); + auto const *dataRef = std::get_if(&term.data); + HS3::Data *data = dataRef ? findData(dataRef->name) : nullptr; + auto *combined = data ? std::get_if(&data->body) : nullptr; + JSONNode const *components = combinedDist ? combinedDist->find("components") : nullptr; + if (!components || !combined) { + continue; + } + for (JSONNode const &component : components->children()) { + JSONNode const *dist = component.find("dist"); + JSONNode const *key = component.find("key"); + if (!dist || !isName(*dist) || !key || !key->is_seq() || key->num_children() != 1) { + continue; + } + std::string const label = key->child(0).val(); + auto ownTerm = + std::find_if(likelihood.terms.begin(), likelihood.terms.end(), [&](HS3::LikelihoodTerm const &other) { + return other.distribution.name == dist->val() && std::holds_alternative(other.data); + }); + if (ownTerm == likelihood.terms.end()) { + continue; + } + auto dataComponent = std::find_if(combined->components.begin(), combined->components.end(), + [&](auto const &c) { return c.key == std::vector{label}; }); + bool hasEntries = false; + if (dataComponent != combined->components.end()) { + HS3::Data const *componentData = findData(dataComponent->data.name); + auto const *unbinned = componentData ? std::get_if(&componentData->body) : nullptr; + hasEntries = componentData && (!unbinned || !unbinned->entries.empty()); + } + if (hasEntries) { + continue; // two contributions, which is not for RooFit to resolve + } + RooJSONFactoryWSTool::warning( + "RooFitHS3: the distribution '" + dist->val() + "' is a component of '" + term.distribution.name + + "' without data and a term of the likelihood '" + likelihood.name + "' with data. It is read as " + + (asChannels ? "the component, with the data of the term." : "the term only.")); + if (asChannels) { + HS3::DataRef termData = std::get(ownTerm->data); + if (!dataOfComponents.insert(termData.name).second) { + HS3::Data const *original = findData(termData.name); + const std::string copyName = data->name + "_" + label; + if (original && !findData(copyName) && doc.data.size() < doc.data.capacity()) { + HS3::Data copy = *original; + copy.name = copyName; + doc.data.push_back(std::move(copy)); + termData.name = copyName; + } + } + if (dataComponent != combined->components.end()) { + dataComponent->data = termData; + } else { + combined->components.push_back({{label}, termData}); + } + termsToRemove.insert(dist->val()); + componentsWithData.insert(dist->val()); + } else { + dropped[term.distribution.name].insert(dist->val()); + labelOf[term.distribution.name][dist->val()] = label; + labels.insert(label); + } + } + } + likelihood.terms.erase(std::remove_if(likelihood.terms.begin(), likelihood.terms.end(), + [&](HS3::LikelihoodTerm const &term) { + return termsToRemove.count(term.distribution.name) > 0; + }), + likelihood.terms.end()); + } + if (asChannels) { + return dropped; + } + + for (auto const &item : doc.rooFit.auxiliaryChannels) { + JSONNode const *node = nullptr; + for (HS3::Distribution const &dist : doc.distributions) { + if (auto const *generic = std::get_if(&dist.body); generic && dist.name == item.first) { + node = generic->node; + } + } + for (auto const &ref : item.second) { + std::string label = ref.name; + if (JSONNode const *components = node ? node->find("components") : nullptr) { + for (JSONNode const &component : components->children()) { + JSONNode const *dist = component.find("dist"); + JSONNode const *key = component.find("key"); + if (dist && isName(*dist) && dist->val() == ref.name && key && key->is_seq() && + key->num_children() == 1) { + label = key->child(0).val(); + } + } + } + dropped[item.first].insert(ref.name); + labelOf[item.first][ref.name] = label; + labels.insert(label); + } + } + if (dropped.empty()) { + return dropped; + } + auto isLabel = [&](std::vector const &key) { return key.size() == 1 && labels.count(key.front()); }; + for (HS3::Likelihood &likelihood : doc.likelihoods) { + std::vector added; + for (HS3::LikelihoodTerm const &term : likelihood.terms) { + auto found = dropped.find(term.distribution.name); + if (found == dropped.end()) { + continue; + } + HS3::CombinedData const *combined = nullptr; + if (auto const *dataRef = std::get_if(&term.data)) { + for (HS3::Data const &data : doc.data) { + if (data.name == dataRef->name) { + combined = std::get_if(&data.body); + } + } + } + for (std::string const &dist : found->second) { + std::string const &label = labelOf[found->first][dist]; + HS3::CombinedDataComponent const *component = nullptr; + if (combined) { + for (HS3::CombinedDataComponent const &c : combined->components) { + if (c.key == std::vector{label}) { + component = &c; + } + } + } + if (std::any_of(likelihood.terms.begin(), likelihood.terms.end(), + [&](auto const &other) { return other.distribution.name == dist; })) { + continue; // a term of its own already + } + if (component) { + added.push_back({{dist}, component->data}); + } else if (std::none_of(likelihood.auxDistributions.begin(), likelihood.auxDistributions.end(), + [&](auto const &ref) { return ref.name == dist; })) { + // Without data, the observed values are the values of the constant observables. + likelihood.auxDistributions.push_back({dist}); + } + } + } + likelihood.terms.insert(likelihood.terms.end(), added.begin(), added.end()); + } + for (HS3::Data &data : doc.data) { + if (auto *combined = std::get_if(&data.body)) { + auto &components = combined->components; + for (auto const &component : components) { + // Not the data sets without entries: they hold no values, and stay data sets of the workspace. + HS3::Data const *componentData = findData(component.data.name); + auto const *unbinned = componentData ? std::get_if(&componentData->body) : nullptr; + if (isLabel(component.key) && !(unbinned && unbinned->entries.empty())) { + dataOfConstraints.insert(component.data.name); + } + } + components.erase(std::remove_if(components.begin(), components.end(), + [&](auto const &component) { return isLabel(component.key); }), + components.end()); + } + } + for (HS3::Domain &domain : doc.domains) { + for (auto &axis : domain.axes) { + if (auto *categorical = std::get_if(&axis)) { + auto &categories = categorical->categories; + categories.erase(std::remove_if(categories.begin(), categories.end(), + [&](std::string const &label) { return labels.count(label) > 0; }), + categories.end()); + } + } + } + return dropped; +} + // Declare the bookkeeping columns of the exported document (see HS3::bookkeepingColumns()): the axes that several // components of a combined data set share, and whose variable in the workspace `ws` is used by nothing else, i.e. has // no clients and is in no set of a ModelConfig. RooAbsData::split() keeps such a column (e.g. an event number) in @@ -602,7 +826,8 @@ void exportAttributes(const RooAbsArg *arg, std::mapattributes()) { // Skip some RooFit internals. The array distribution tag is restored from the type on import. - if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object" || attr == arrayDistTag) + if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object" || attr == arrayDistTag || + attr == helperObjectTag) continue; initialize(); out->tags.push_back(attr); @@ -622,6 +847,20 @@ HS3::BinnedAxis RooFit::HS3::toBinnedAxis(std::string const &name, RooAbsBinning return {name, binning.lowBound(), binning.highBound(), toBinning(binning)}; } +HS3::ContinuousAxis RooFit::HS3::toContinuousAxis(RooRealVar const &var) +{ + RooJSONFactoryWSTool::testValidName(var.GetName(), false); + HS3::ContinuousAxis axis; + axis.name = var.GetName(); + if (!RooNumber::isInfinite(var.getMin())) { + axis.min = var.getMin(); + } + if (!RooNumber::isInfinite(var.getMax())) { + axis.max = var.getMax(); + } + return axis; +} + HS3::NameOrNumber RooFit::HS3::toNameOrNumber(RooAbsArg const &arg) { if (isLiteralConstVar(arg)) { @@ -756,16 +995,15 @@ std::vector permute(std::vector const &values, std::vectorcontains(axis.name)) { - var.setRange(*axis.min, *axis.max); - } else if (var.getMin() != *axis.min || var.getMax() != *axis.max) { + var.setRange(min, max); + } else if (var.getMin() != min || var.getMax() != max) { std::stringstream ss; - ss << "RooFitHS3: the range [" << *axis.min << ", " << *axis.max << "] of the axis '" << axis.name - << "' of the dataset '" << dataName << "' differs from the range [" << var.getMin() << ", " << var.getMax() - << "] of the variable."; + ss << "RooFitHS3: the range [" << min << ", " << max << "] of the axis '" << axis.name << "' of the dataset '" + << dataName << "' differs from the range [" << var.getMin() << ", " << var.getMax() << "] of the variable."; RooJSONFactoryWSTool::warning(ss.str()); } } @@ -818,7 +1056,8 @@ std::unique_ptr createUnbinnedData(std::string const &name, HS3::Unb } // Create a RooAbsData (binned or unbinned) from its description. -std::unique_ptr loadData(HS3::Data const &input, RooWorkspace &workspace, HS3::DomainRanges const *ranges) +std::unique_ptr loadData(HS3::Data const &input, RooWorkspace &workspace, HS3::DomainRanges const *ranges, + std::string const *soleVariate = nullptr) { RooJSONFactoryWSTool::testValidName(input.name, true); @@ -844,7 +1083,27 @@ std::unique_ptr loadData(HS3::Data const &input, RooWorkspace &works } return createUnbinnedData(input.name, *unbinned, workspace, ranges); } - RooJSONFactoryWSTool::error("datasets of type \"point\" like '" + input.name + "' are not supported yet"); + // Point data is the observation of a single variable. With its axis, it is the same as unbinned data with a single + // entry, and becomes a RooDataSet with one entry. + auto const &point = std::get(input.body); + if (point.uncertainty) { + RooJSONFactoryWSTool::warning("RooFitHS3: the \"uncertainty\" of the dataset '" + input.name + + "' is ignored: HS3 doesn't define how it enters a likelihood."); + } + HS3::UnbinnedData unbinned; + if (point.axes.empty() && soleVariate) { + // Without an axis, the value is the one of the single variate of the paired distribution (HS3 0.3). + unbinned.axes.emplace_back(HS3::ContinuousAxis{*soleVariate, std::nullopt, std::nullopt}); + } else if (point.axes.size() == 1) { + unbinned.axes.emplace_back(point.axes.front()); + } else if (point.axes.empty()) { + RooJSONFactoryWSTool::error("the point data set '" + input.name + + "' has no axis, and it is not paired with a distribution that has a single variate"); + } else { + RooJSONFactoryWSTool::error("the point data set '" + input.name + "' must have exactly one axis"); + } + unbinned.entries.push_back(point.value); + return createUnbinnedData(input.name, unbinned, workspace, ranges); } // The version of ROOT that writes the document, with dots only such that it can be read automatically. @@ -984,9 +1243,68 @@ void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysi } } +// In HS3, the constraint terms of a model are terms of the likelihood; in RooFit, as HistFactory builds its models, a +// constraint is a factor of every channel whose parameters it constrains, which makes the pdf of the model the +// sampling distribution of the auxiliary measurements as well. The export takes the constraints out of the channels +// (writeLikelihoodTermsOnce()); this puts them back: a constraint term with a global observable (a constant variable +// that the model itself does not depend on) becomes a factor of every channel that is a product and depends on one +// of its other parameters. An array of constraints goes to its channels as a whole. The constraints that +// found a channel are removed from `constraints`; the others stay external constraints of the ModelConfig, e.g. a +// penalty term without an observation, like the regularization of an unfolding, or a constraint of a model whose +// channels are not products. +void attachConstraintsToChannels(RooAbsPdf &pdf, RooArgSet &constraints) +{ + RooArgSet channels; + if (auto *simPdf = dynamic_cast(&pdf)) { + for (auto const &state : simPdf->indexCat()) { + if (RooAbsPdf *channel = simPdf->getPdf(state.first)) { + channels.add(*channel, /*silent=*/true); + } + } + } else { + channels.add(pdf); + } + RooArgSet attached; + for (auto *constraint : static_range_cast(constraints)) { + // An array of constraints enters as a whole: its components have no names in a document, so that a channel + // could not be written with them as factors. + std::vector terms{constraint}; + // Decided for all terms before anything is changed, so that a constraint is not attached in part. + std::set found; + std::vector> factors; + for (auto *channel : static_range_cast(channels)) { + auto *product = dynamic_cast(channel); + if (!product || channel == constraint) { + continue; + } + for (RooAbsPdf *term : terms) { + if (product->pdfList().find(*term)) { + found.insert(term); + } + } + for (RooAbsPdf *term : constraintsOfChannel(*channel, terms, channels)) { + if (!product->pdfList().find(*term)) { + factors.emplace_back(product, term); + } + found.insert(term); + } + } + if (terms.empty() || found.size() != terms.size()) { + continue; + } + for (auto const &[channel, term] : factors) { + channel->addPdfs(RooArgSet{*term}); + } + attached.add(*constraint); + } + constraints.remove(attached); +} + // Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWorkspace &workspace, - std::vector> &datasets, std::set const &bookkeepingColumns) + std::vector> &datasets, std::set const &bookkeepingColumns, + std::map, std::string> const &inlineData, + std::set const &auxiliaryMeasurements) { // if this is a toplevel pdf, also create a modelConfig for it std::string const &analysisName = analysis.name; @@ -1008,6 +1326,22 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo } HS3::Likelihood const &likelihood = *foundLikelihood; + // The data set of a term: the one it refers to, or the one made from a number given in-line. + std::map inlineRefs; + for (std::size_t i = 0; i < likelihood.terms.size(); ++i) { + auto found = inlineData.find({likelihood.name, i}); + if (found != inlineData.end()) { + inlineRefs[&likelihood.terms[i]] = HS3::DataRef{found->second}; + } + } + auto dataRefOf = [&](HS3::LikelihoodTerm const &term) -> HS3::DataRef const * { + if (auto const *ref = std::get_if(&term.data)) { + return ref; + } + auto found = inlineRefs.find(&term); + return found != inlineRefs.end() ? &found->second : nullptr; + }; + auto checkPdf = [&](RooAbsArg const *arg, std::string const &name, std::string const &what) { requirePdf(arg, "the " + what + " '" + name + "' of analysis '" + analysisName + "'"); }; @@ -1059,9 +1393,12 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo [&](auto const &term) { return term.distribution.name == modelPdfName; })) { modelTree.reset(modelPdf->getComponents()); } + // So is a component of a combined distribution that is read as a constraint term (`auxiliaryMeasurements`, see + // readAuxiliaryChannelsAsConstraints()), whatever the document says about the ModelConfig. auto isConstraintTermOfModel = [&](RooAbsPdf const &listed, HS3::LikelihoodTerm const &term) { - auto const *dataRef = std::get_if(&term.data); - if (!modelTree || !dataRef || modelTree->find(listed.GetName())) { + auto const *dataRef = dataRefOf(term); + const bool auxiliary = auxiliaryMeasurements.count(listed.GetName()) > 0; + if (!dataRef || (!auxiliary && (!modelTree || modelTree->find(listed.GetName())))) { return false; } return std::any_of(datasets.begin(), datasets.end(), @@ -1082,7 +1419,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo nllTerms.push_back(&term); continue; } - auto const *dataRef = std::get_if(&term.data); + auto const *dataRef = dataRefOf(term); if (!dataRef) { continue; } @@ -1142,7 +1479,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo RooArgSet observables; for (HS3::LikelihoodTerm const *termPtr : nllTerms) { HS3::LikelihoodTerm const &term = *termPtr; - auto const *dataRef = std::get_if(&term.data); + auto const *dataRef = dataRefOf(term); if (!dataRef) { continue; } @@ -1169,7 +1506,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo // A combined distribution paired with a combined data set must have a sub-data set for every component. for (auto const &term : likelihood.terms) { - auto const *dataRef = std::get_if(&term.data); + auto const *dataRef = dataRefOf(term); auto *simPdf = dynamic_cast(workspace.pdf(term.distribution.name)); if (!dataRef || !simPdf) { continue; @@ -1210,6 +1547,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo mc->SetPdf(*pdf); extConstraints.add(constraintsWithData, /*silent=*/true); + attachConstraintsToChannels(*pdf, extConstraints); if (!extConstraints.empty()) mc->SetExternalConstraints(extConstraints); @@ -1501,10 +1839,11 @@ void sortByName(T &coll) /// getVal(normSet), or the integral returned by createIntegral(). These live /// outside the workspace and are not evidence that the object is a sub-node of /// a bigger model (see https://github.com/root-project/root/issues/23221). +// Whether no object of the workspace uses `arg`. Helpers that are not written don't count as users. bool isTopLevel(RooAbsArg const &arg, RooWorkspace const &ws) { for (RooAbsArg const *client : arg.clients()) { - if (ws.components().containsInstance(*client)) { + if (ws.components().containsInstance(*client) && !isSkippedHelper(*client, ws)) { return false; } } @@ -1551,8 +1890,132 @@ std::unique_ptr RooFit::HS3::createDataHist(std::string const &name return dh; } +std::vector +constraintsOfChannel(RooAbsPdf const &channel, std::vector const &terms, RooArgSet const &channels) +{ + std::vector out; + for (RooAbsPdf *candidate : terms) { + for (RooAbsPdf *term : {candidate}) { + if (term == &channel || channels.containsInstance(*term)) { + continue; + } + RooArgSet leaves; + term->leafNodeServerList(&leaves); + RooArgSet parameters; + bool hasGlobalObservable = false; + for (RooAbsArg *leaf : leaves) { + auto *var = dynamic_cast(leaf); + if (!var) { + continue; + } + // Whether a global observable is constant yet depends on where the document has its value: as a + // parameter, or in the data set of the constraint, which is read later. + if (std::none_of(channels.begin(), channels.end(), + [&](RooAbsArg const *other) { return other->dependsOn(*var); })) { + hasGlobalObservable = true; + } else { + parameters.add(*var); + } + } + if (hasGlobalObservable && !parameters.empty() && channel.dependsOn(parameters) && + std::find(out.begin(), out.end(), term) == out.end()) { + out.push_back(term); + } + } + } + return out; +} + +RooAbsPdf & +channelWithConstraints(RooJSONFactoryWSTool &tool, RooAbsPdf &channel, std::vector const &constraints) +{ + if (constraints.empty()) { + return channel; + } + RooArgList factors; + for (RooAbsPdf *constraint : constraints) { + factors.add(*constraint); + } + factors.add(channel); + return tool.wsEmplace(std::string(channel.GetName()) + "_with_constraints", factors); +} + +bool isSkippedHelper(RooAbsArg const &arg, RooWorkspace const &ws) +{ + if (!arg.getAttribute(helperObjectTag)) { + return false; + } + // The HS3 object that the helper stands for is written: an array distribution. + if (arg.getAttribute(arrayDistTag) && RooJSONFactoryWSTool::config().exportArrayDist) { + return false; + } + for (RooAbsArg const *client : arg.clients()) { + if (ws.components().containsInstance(*client) && !isSkippedHelper(*client, ws)) { + return false; + } + } + for (TObject const *obj : ws.allGenericObjects()) { + auto const *mc = dynamic_cast(obj); + if (!mc) { + continue; + } + if (mc->GetPdf() == &arg || mc->GetPriorPdf() == &arg || + (mc->GetExternalConstraints() && mc->GetExternalConstraints()->containsInstance(arg))) { + return false; + } + } + return true; +} + RooJSONFactoryWSTool::RooJSONFactoryWSTool(RooWorkspace &ws) : _workspace{ws} {} +std::vector RooJSONFactoryWSTool::otherLikelihoodTerms(std::string const &dist) const +{ + std::vector out; + if (!_import) { + return out; + } + auto add = [&](std::string const &name) { + if (name != dist && std::find(out.begin(), out.end(), name) == out.end()) { + out.push_back(name); + } + }; + for (auto const &likelihood : _import->document.likelihoods) { + if (std::none_of(likelihood.terms.begin(), likelihood.terms.end(), + [&](auto const &term) { return term.distribution.name == dist; })) { + continue; + } + for (auto const &term : likelihood.terms) { + add(term.distribution.name); + } + for (auto const &aux : likelihood.auxDistributions) { + add(aux.name); + } + } + return out; +} + +bool RooJSONFactoryWSTool::isConstraintReadAsChannel(RooAbsPdf const &pdf) const +{ + if (_componentsWithData.count(pdf.GetName())) { + return true; + } + // A component of an array distribution, which is the product of its components in RooFit. + for (RooAbsArg const *client : pdf.clients()) { + if (client->getAttribute(arrayDistTag) && _componentsWithData.count(client->GetName())) { + return true; + } + } + return false; +} + +bool RooJSONFactoryWSTool::isAuxiliaryChannelReadAsConstraint(std::string const &combinedDist, + std::string const &component) const +{ + auto found = _auxiliaryChannelsAsConstraints.find(combinedDist); + return found != _auxiliaryChannelsAsConstraints.end() && found->second.count(component); +} + RooJSONFactoryWSTool::~RooJSONFactoryWSTool() = default; HS3::Version RooJSONFactoryWSTool::hs3Version() const @@ -1730,7 +2193,17 @@ void RooJSONFactoryWSTool::exportVariables(const RooArgSet &allElems, HS3::Param std::string RooJSONFactoryWSTool::exportTransformed(const RooAbsReal *original, const std::string &suffix, const std::string &formula) { - std::string newname = std::string(original->GetName()) + suffix; + // A literal constant is spelled as an identifier, which a parameter must not be mistaken for when reading. + if (!isLiteralConstVar(*original) && isLiteralIdentifier(original->GetName())) { + error("cannot write a function of '" + std::string(original->GetName()) + + "': names of the form 'const_' are reserved for functions of literal constants"); + } + std::string newname = transformedBaseName(*original) + suffix; + // HS3 0.3 (HS3 PR #154) only allows expressions to refer to parameters, variables and functions. + if (dynamic_cast(original)) { + error("the parameter '" + std::string(original->GetName()) + + "' would have to be written as an expression that refers to a pdf, which HS3 doesn't allow"); + } // Another object that shares the parameter might have exported the helper already. auto &functions = _export->doc.functions; if (std::find_if(functions.begin(), functions.end(), [&](auto const &f) { return f.name == newname; }) != @@ -1739,7 +2212,7 @@ std::string RooJSONFactoryWSTool::exportTransformed(const RooAbsReal *original, } RooFit::Detail::JSONNode &trafo_node = appendNamedChild((*_export->root)["functions"], newname); trafo_node["type"] << "generic"; - trafo_node["expression"] << TString::Format(formula.c_str(), original->GetName()).Data(); + trafo_node["expression"] << TString::Format(formula.c_str(), transformedArgument(*original).c_str()).Data(); functions.push_back({newname, {"generic", nullptr}}); this->setAttribute(newname, "roofit_skip"); // this function should not be imported back in return newname; @@ -1972,7 +2445,7 @@ void RooJSONFactoryWSTool::exportArrayDists() // Exporting the servers may add to the registry, so the components are copied. const std::vector components = next->second; JSONNode &elem = appendNamedChild((*_export->root)["distributions"], arrayName); - writeArrayDist(elem, arrayName, components, _export->constraintObservables); + writeArrayDist(*this, elem, arrayName, components, _export->constraintObservables); recordGenericObject(_export->doc, true, arrayName, elem); for (RooAbsPdf const *pdf : components) { exportObjects(pdf->servers()); @@ -2002,14 +2475,20 @@ void RooJSONFactoryWSTool::recordConstraintTerms(std::string const &pdfName, std } } -// Write the data set "_data" with the observed values of the observables of the constraint term `term` (an +// Write the data set `dataName` with the observed values of the observables of the constraint term `term` (an // array distribution written in this export, or a single constraint pdf) and return true, or return false if the // observables of the term are not all global observables of the model. -bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgSet const *globalObservables) +bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgSet const *globalObservables, + std::string const &dataName) { std::vector components; if (auto found = _export->arrayComponents.find(term); found != _export->arrayComponents.end()) { components = found->second; + } else if (auto const *product = dynamic_cast(_workspace.pdf(term))) { + // A product of constraint terms: its factors. + for (RooAbsArg const *factor : product->pdfList()) { + components.push_back(static_cast(factor)); + } } else if (RooAbsPdf *pdf = _workspace.pdf(term)) { components.push_back(pdf); } @@ -2024,7 +2503,6 @@ bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgS if (observables.empty()) { return false; } - const std::string dataName = term + "_data"; if (!_export->writtenConstraintData.insert(dataName).second) { return true; } @@ -2038,22 +2516,23 @@ bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgS if (!var) { error("the observable '" + name + "' of the constraint term '" + term + "' is not in the workspace"); } + // A range without a bound leaves the bound out, which means that the axis is unbounded there. HS3::ContinuousAxis axis; axis.name = name; - // A range without a bound gets the largest representable value, so that other engines see a finite axis. - axis.min = -std::numeric_limits::max(); - axis.max = std::numeric_limits::max(); if (auto const *rrv = dynamic_cast(var)) { - if (std::isfinite(rrv->getMin())) { - axis.min = rrv->getMin(); - } - if (std::isfinite(rrv->getMax())) { - axis.max = rrv->getMax(); - } + axis = HS3::toContinuousAxis(*rrv); } data.axes.emplace_back(std::move(axis)); data.entries.push_back(var->getVal()); } + // The observed value of a single observable is point data. + if (data.axes.size() == 1) { + HS3::PointData point; + point.value = data.entries.front(); + point.axes.push_back(std::get(data.axes.front())); + _export->doc.data.push_back({dataName, std::move(point)}); + return true; + } _export->doc.data.push_back({dataName, std::move(data)}); return true; } @@ -2475,6 +2954,21 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) } } + // A single unweighted entry of a single variable is point data. + if (variables.size() == 1 && data.numEntries() == 1) { + data.get(0); + auto const *var = dynamic_cast(variables[0]); + if (var && (!data.isWeighted() || data.weight() == 1.)) { + recordDomains(_export->domains, *var); + HS3::PointData point; + point.value = var->getVal(); + point.axes.push_back(HS3::toContinuousAxis(*var)); + output.body = std::move(point); + _export->doc.data.push_back(std::move(output)); + return; + } + } + // this really is an unbinned dataset HS3::UnbinnedData unbinned; for (RooAbsArg const *arg : variables) { @@ -2484,7 +2978,7 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) } else { // HS3 forbids binning information on the axes of unbinned data. auto const &var = static_cast(*arg); - unbinned.axes.emplace_back(HS3::ContinuousAxis{var.GetName(), var.getMin(), var.getMax()}); + unbinned.axes.emplace_back(HS3::toContinuousAxis(var)); } } std::vector weightVals; @@ -2637,6 +3131,8 @@ struct ChannelConstraint { std::string channel; RooAbsPdf const *constraint = nullptr; bool histFactory = false; ///< Whether the channel is written as a histfactory_dist. + /// Whether the channel consists of constraint terms only; `constraint` is then the channel itself. + bool wholeChannel = false; }; // The constraint terms of the pdf `pdf` of a ModelConfig with the observables `observables`, which are written as @@ -2684,6 +3180,14 @@ constraintsOfChannels(RooWorkspace const &ws, RooAbsPdf const &pdf, RooArgSet co continue; } const bool histFactory = isHistFactoryChannel(prod->GetName()); + // A channel that consists of constraint terms only is an auxiliary measurement that is part of the model. It + // is written as that channel, once, and not as constraint terms of the likelihood. + if (!histFactory && !prod->pdfList().empty() && + std::all_of(prod->pdfList().begin(), prod->pdfList().end(), + [&](RooAbsArg const *factor) { return constraints->find(*factor) != nullptr; })) { + out.push_back({prod->GetName(), prod, false, true}); + continue; + } if (!histFactory) { // Being a component of a RooSimultaneous, i.e. of a combined distribution, is not a use. bool usedElsewhere = false; @@ -2734,7 +3238,9 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc if (dataName) { likelihood.terms.push_back({{pdf->GetName()}, HS3::DataRef{*dataName}}); } else { - likelihood.terms.push_back({{pdf->GetName()}, 0.0}); + // Without data, the likelihood is a pure model, which is written with "data": null. + likelihood.terms.push_back({{pdf->GetName()}, std::monostate{}}); + likelihood.pureModel = true; } // The additional terms of the likelihood: the external constraints of the ModelConfig, and the constraint terms @@ -2761,6 +3267,8 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc } } } + // The channels of the model that consist of constraint terms only, see constraintsOfChannels(). + std::vector auxiliaryChannelsOfModel; std::vector constraintTerms; if (mc.GetExternalConstraints()) { for (const auto &constr : *mc.GetExternalConstraints()) { @@ -2807,11 +3315,14 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc for (ChannelConstraint const &item : constraintsOfChannels(_workspace, *pdf, *observables, isHistFactoryChannel, renamedOfPdf != _renamedComponents.end() ? &renamedOfPdf->second : nullptr)) { + if (item.wholeChannel) { + auxiliaryChannelsOfModel.push_back(item.channel); + continue; + } + // A component of an array distribution is never referred to by itself: the array is the term. std::string term = item.constraint->GetName(); - if (item.histFactory) { - if (auto found = _export->arrayOfComponent.find(term); found != _export->arrayOfComponent.end()) { - term = found->second; - } + if (auto found = _export->arrayOfComponent.find(term); found != _export->arrayOfComponent.end()) { + term = found->second; } addUnique(termsOfChannel[item.channel], term); } @@ -2845,9 +3356,120 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc } } } + // An array of constraints with its data set is a measurement like the others: if the model is a combined + // distribution, it is written as a component of it, paired with its data set as a component of the combined data + // set. The combined distribution is then the sampling distribution of all measurements, auxiliary or not. Its + // label is the name of the array. Constraints that are written one by one stay terms of the likelihood: there + // can be hundreds, which is not what an index category is for. So do the constraints of models whose index has + // several columns. + auto const *simPdf = dynamic_cast(pdf); + // Looked up when needed: writing the data set of a constraint term adds to the data sets of the document. + auto findCombinedData = [&]() -> HS3::CombinedData * { + for (HS3::Data &data : doc.data) { + if (dataName && data.name == *dataName) { + return std::get_if(&data.body); + } + } + return nullptr; + }; + RooCategory const *indexColumn = nullptr; + if (simPdf && findCombinedData()) { + std::vector columns = selectionColumns(simPdf->indexCat()); + if (columns.size() == 1) { + indexColumn = columns.front(); + } + } + auto addAuxiliaryChannel = [&](std::string const &term) { + if (indexColumn->hasLabel(term)) { + error("cannot write the constraint term '" + term + "' as a component of the combined distribution '" + + simPdf->GetName() + "': its index category '" + indexColumn->GetName() + "' has a state of that name"); + } + auto addOnce = [&](std::vector &list) { + if (std::find(list.begin(), list.end(), term) == list.end()) { + list.push_back(term); + return true; + } + return false; + }; + addOnce(_export->auxiliaryLabels[indexColumn->GetName()]); + if (addOnce(_export->auxiliaryChannels[simPdf->GetName()])) { + doc.rooFit.auxiliaryChannels[simPdf->GetName()].push_back({term}); + } + HS3::CombinedData *combinedData = findCombinedData(); + if (std::none_of(combinedData->components.begin(), combinedData->components.end(), + [&](auto const &component) { return component.key == std::vector{term}; })) { + combinedData->components.push_back({{term}, HS3::DataRef{*dataName + "_" + term}}); + } + }; + // A channel of the model that consists of constraint terms is an auxiliary measurement, whose observed values are + // the values of its global observables if the data set has no entries for it. It is written like the arrays + // above: with a data set of the observed values, and marked for RooFit, which can read it as constraint terms. + for (std::string const &channel : auxiliaryChannelsOfModel) { + if (!indexColumn || likelihood.pureModel) { + continue; + } + std::string label; + for (auto const &state : simPdf->indexCat()) { + RooAbsPdf const *component = simPdf->getPdf(state.first); + if (component && channel == component->GetName()) { + label = state.first; + } + } + if (label.empty()) { + continue; + } + HS3::CombinedData *combinedData = findCombinedData(); + auto component = std::find_if(combinedData->components.begin(), combinedData->components.end(), + [&](auto const &c) { return c.key == std::vector{label}; }); + const std::string observed = + component != combinedData->components.end() ? component->data.name : *dataName + "_" + label; + // A data set without entries for the channel is replaced by the observed values. + auto written = std::find_if(doc.data.begin(), doc.data.end(), [&](auto const &d) { return d.name == observed; }); + bool hasEntries = false; + if (written != doc.data.end()) { + auto const *unbinned = std::get_if(&written->body); + hasEntries = !unbinned || !unbinned->entries.empty(); + } + if (!hasEntries) { + std::optional empty; + if (written != doc.data.end()) { + empty = std::move(*written); + doc.data.erase(written); + } + if (exportConstraintData(channel, mc.GetGlobalObservables(), observed)) { + combinedData = findCombinedData(); + if (std::none_of(combinedData->components.begin(), combinedData->components.end(), + [&](auto const &c) { return c.key == std::vector{label}; })) { + combinedData->components.push_back({{label}, HS3::DataRef{observed}}); + } + } else { + if (empty) { + doc.data.push_back(std::move(*empty)); + } + warning("RooFitHS3: the channel '" + channel + "' of '" + simPdf->GetName() + + "' consists of constraint terms, but has no data, and its observables are not all global " + "observables of the ModelConfig: it is written without an observation."); + continue; + } + } + auto &marked = doc.rooFit.auxiliaryChannels[simPdf->GetName()]; + if (std::none_of(marked.begin(), marked.end(), [&](auto const &ref) { return ref.name == channel; })) { + marked.push_back({channel}); + } + } for (std::string const &term : constraintTerms) { - if (config().exportConstraintData && exportConstraintData(term, mc.GetGlobalObservables())) { - likelihood.terms.push_back({{term}, HS3::DataRef{term + "_data"}}); + // A pure model has no data at all, so the observed values of its constraint terms are not written as data sets + // either: they are the constant parameters of auxiliary distributions. + // A component of a combined data set belongs to that data set alone, and is named after it like the others. + const bool asChannel = indexColumn && _export->arrayComponents.count(term); + const std::string constraintData = asChannel ? *dataName + "_" + term : term + "_data"; + if (!likelihood.pureModel && config().exportConstraintData && + exportConstraintData(term, mc.GetGlobalObservables(), constraintData)) { + if (asChannel) { + addAuxiliaryChannel(term); + } else { + likelihood.terms.push_back({{term}, HS3::DataRef{constraintData}}); + } } else { likelihood.auxDistributions.push_back({term}); } @@ -2921,11 +3543,29 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) std::vector allpdfs; for (auto &arg : _workspace.allPdfs()) { if (isTopLevel(*arg, _workspace)) { - if (auto *pdf = dynamic_cast(arg)) { + auto *pdf = dynamic_cast(arg); + if (pdf && !isSkippedHelper(*pdf, _workspace)) { allpdfs.push_back(pdf); } } } + // A Gaussian constraint is written with its global observable as "x", also if that is its mean in RooFit (see + // setConstraintObservable()). Which one it is must be known before any pdf is written: the pdfs are written in the + // order of their names, and a constraint can come before the model that uses it. + for (TObject *obj : _workspace.allGenericObjects()) { + auto const *mc = dynamic_cast(obj); + RooArgSet const *globs = mc ? mc->GetGlobalObservables() : nullptr; + if (!globs) { + continue; + } + for (RooAbsArg const *arg : _workspace.components()) { + auto const *gauss = dynamic_cast(arg); + if (gauss && globs->find(gauss->getMean()) && !globs->find(gauss->getX()) && + !constraintObservable(gauss->GetName())) { + setConstraintObservable(*gauss, gauss->getMean().GetName()); + } + } + } sortByName(allpdfs); exportObjects(allpdfs); @@ -2933,7 +3573,8 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) std::vector allfuncs; for (auto &arg : _workspace.allFunctions()) { if (isTopLevel(*arg, _workspace)) { - if (auto *func = dynamic_cast(arg)) { + auto *func = dynamic_cast(arg); + if (func && !isSkippedHelper(*func, _workspace)) { allfuncs.push_back(func); } } @@ -3004,6 +3645,15 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) this->exportVariables(snapshotSorted, doc.parameterPoints.back(), true); } } + // The constraint terms that are components of a combined distribution have a label of their own. + for (auto const &item : _export->auxiliaryLabels) { + auto &categories = _export->categories[item.first].categories; + for (std::string const &label : item.second) { + if (std::find(categories.begin(), categories.end(), label) == categories.end()) { + categories.push_back(label); + } + } + } // The categories are categorical axes of the "default_domain", which must not list them as continuous. auto &defaultAxes = _export->domains[HS3::defaultDomainName]; for (auto const &item : _export->categories) { @@ -3022,6 +3672,23 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) } } + // The constraint terms that are components of a combined distribution, see exportSingleModelConfig(). The + // combined distributions are written as generic distributions, so the components are added to the JSON tree. + for (auto const &item : _export->auxiliaryChannels) { + JSONNode const *distributions = n.find("distributions"); + JSONNode const *dist = distributions ? findNamedChild(*distributions, item.first) : nullptr; + if (!dist || !dist->has_child("components")) { + error("cannot write the constraint terms of '" + item.first + + "' as its components: it was not written as a combined distribution"); + } + JSONNode &components = const_cast(*dist)["components"]; + for (std::string const &term : item.second) { + JSONNode &component = components.append_child().set_map(); + component["key"].fill_seq(std::vector{term}); + component["dist"] << term; + } + } + // The exporters have written the generic distributions to the JSON tree. Now that the tree has all of them, the // document refers to their nodes, like a document that was read. if (JSONNode const *distributions = n.find("distributions")) { @@ -3166,6 +3833,21 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) HS3::Diagnostics diagnostics; HS3::upgradeDocument(n, diagnostics, options); HS3Document parsed = HS3::parseDocument(n, diagnostics, options); + _auxiliaryChannelsAsConstraints.clear(); + // The data sets of the components that are read as constraint terms: their values end up in the global + // observables, and they are not data sets of the workspace. + std::set dataOfConstraints; + std::set componentsWithData; + // As the document has them, before the components that are read as constraint terms are taken out of it. + _componentsWithData = HS3::componentsWithData(parsed); + _auxiliaryChannelsAsConstraints = readAuxiliaryChannelsAsConstraints(parsed, config().importConstraintsAsChannels, + dataOfConstraints, componentsWithData); + _componentsWithData.insert(componentsWithData.begin(), componentsWithData.end()); + for (auto const &item : _auxiliaryChannelsAsConstraints) { + for (std::string const &component : item.second) { + _componentsWithData.erase(component); + } + } std::optional validated; if (!HS3::hasErrors(diagnostics)) { HS3::ValidateOptions validateOptions; @@ -3179,7 +3861,12 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) // The state of the import refers to the document, so it must not survive it, not even if there is an exception. struct ImportScope { RooJSONFactoryWSTool &tool; - ~ImportScope() { tool._import.reset(); } + ~ImportScope() + { + tool._import.reset(); + tool._auxiliaryChannelsAsConstraints.clear(); + tool._componentsWithData.clear(); + } } importScope{*this}; _import = std::make_unique(doc); @@ -3248,14 +3935,94 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) // The components of combined data sets are moved out of the "datasets" vector when the combined data set is // assembled, so they don't get imported on their own. The ROOT-internal form of older files (combineDatasets()) // is handled after the analyses, because their likelihoods refer to the components. + // A number given in-line in the "data" of a likelihood, or a point data set without axes, is the value of the single + // variate of the paired distribution (HS3 0.3). If that can't be determined, the document is rejected. + auto soleVariateOf = [&](std::string const &distribution, std::string const &what) { + std::set const v = HS3::variatesOf(doc, distribution); + if (v.size() != 1) { + error("the variate of the distribution '" + distribution + "', which " + what + + " is paired with, can't be determined, since it doesn't have a single variate"); + } + return *v.begin(); + }; + std::map pointDataVariates; + std::map, std::string> inlineData; + std::vector inlineDataSets; + // Older versions of RooFit wrote "data": [0], with the number 0 as the only entry, for a ModelConfig without data. + // In this exact form, it is no observation in documents before 0.3 (HS3 PR #154). RooFit now writes "data": null. + auto isNoDataPlaceholder = [&](HS3::Likelihood const &likelihood, double value) { + return value == 0. && likelihood.terms.size() == 1 && doc.metadata.version < HS3::Version{0, 3}; + }; + for (auto const &likelihood : doc.likelihoods) { + for (std::size_t i = 0; i < likelihood.terms.size(); ++i) { + auto const &term = likelihood.terms[i]; + if (auto const *value = std::get_if(&term.data)) { + if (isNoDataPlaceholder(likelihood, *value)) { + continue; + } + std::string const what = "the number in the \"data\" of the likelihood '" + likelihood.name + "'"; + HS3::PointData point; + point.value = *value; + point.axes.push_back( + HS3::ContinuousAxis{soleVariateOf(term.distribution.name, what), std::nullopt, std::nullopt}); + auto isTaken = [&](std::string const &candidate) { + return std::any_of(doc.data.begin(), doc.data.end(), + [&](HS3::Data const &d) { return d.name == candidate; }) || + std::any_of(inlineDataSets.begin(), inlineDataSets.end(), + [&](HS3::Data const &d) { return d.name == candidate; }); + }; + std::string name = term.distribution.name + "_data"; + for (std::size_t k = 1; isTaken(name); ++k) { + name = term.distribution.name + "_data_" + std::to_string(k); + } + inlineData[{likelihood.name, i}] = name; + inlineDataSets.push_back({name, std::move(point)}); + continue; + } + auto const *refPtr = std::get_if(&term.data); + if (!refPtr) { + continue; // no data in a pure model + } + auto const &ref = *refPtr; + for (auto const &data : doc.data) { + auto const *point = std::get_if(&data.body); + if (data.name != ref.name || !point) { + continue; + } + if (!point->axes.empty()) { + // The axis of point data is named like the variate of the paired distribution. + std::set const v = HS3::variatesOf(doc, term.distribution.name); + if (v.size() == 1 && point->axes.front().name != *v.begin()) { + error("the axis '" + point->axes.front().name + "' of the point data set '" + data.name + + "' is not the variate '" + *v.begin() + "' of the distribution '" + term.distribution.name + + "' it is paired with"); + } + continue; + } + std::string const variate = soleVariateOf(term.distribution.name, "the point data set '" + data.name + "'"); + auto inserted = pointDataVariates.emplace(data.name, variate); + if (!inserted.second && inserted.first->second != variate) { + error("the point data set '" + data.name + + "' without axes is paired with distributions of different variates, '" + inserted.first->second + + "' and '" + variate + "'"); + } + } + } + } + std::vector> datasets; - datasets.reserve(doc.data.size()); + datasets.reserve(doc.data.size() + inlineDataSets.size()); for (auto const &data : doc.data) { // Combined data sets are assembled from the others once all of them are loaded. if (!std::holds_alternative(data.body)) { - datasets.push_back(loadData(data, _workspace, &_import->ranges)); + auto variate = pointDataVariates.find(data.name); + datasets.push_back(loadData(data, _workspace, &_import->ranges, + variate != pointDataVariates.end() ? &variate->second : nullptr)); } } + for (auto const &data : inlineDataSets) { + datasets.push_back(loadData(data, _workspace, &_import->ranges)); + } assembleCombinedData(doc, _workspace, datasets); // Now, read in analyses and likelihoods if there are any @@ -3281,12 +4048,33 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) ". HS3 0.3 requires distinct variates; the document is read anyway, since it declares an older " "version."); } + if (!(doc.metadata.version < HS3::Version{0, 3})) { + std::vector notes; + for (std::string const &message : HS3::unlistedConstraints(doc, ¬es)) { + error("invalid HS3: " + message + + ". The \"constraints\" of a modifier add no term to the likelihood, which has to list them."); + } + for (std::string const ¬e : notes) { + warning("RooFitHS3: " + note + "."); + } + } for (auto const &analysis : doc.analyses) { - importAnalysis(doc, analysis, _workspace, datasets, bookkeepingColumns); + std::set auxiliaryMeasurements; + for (auto const &item : _auxiliaryChannelsAsConstraints) { + auxiliaryMeasurements.insert(item.second.begin(), item.second.end()); + } + importAnalysis(doc, analysis, _workspace, datasets, bookkeepingColumns, inlineData, auxiliaryMeasurements); } combineDatasets(doc, datasets); + // Several analyses can share a ModelConfig, of which only the first one consumes the data of its constraint + // terms. + for (auto &d : datasets) { + if (d && dataOfConstraints.count(d->GetName())) { + d.reset(); + } + } for (auto const &d : datasets) { if (d) { diff --git a/roofit/hs3/test/CMakeLists.txt b/roofit/hs3/test/CMakeLists.txt index 5edcc83457694..9e43a3ccd841a 100644 --- a/roofit/hs3/test/CMakeLists.txt +++ b/roofit/hs3/test/CMakeLists.txt @@ -1,4 +1,5 @@ ROOT_ADD_GTEST(testRooFitHS3 testRooFitHS3.cxx LIBRARIES RooFitCore RooFit RooFitHS3 RooStats HistFactory) +ROOT_ADD_GTEST(testHS3AuxiliaryMeasurements testHS3AuxiliaryMeasurements.cxx LIBRARIES RooFitCore RooFit RooFitHS3 RooStats HistFactory) if(test_roofit_hs3testsuite) include(FetchContent) diff --git a/roofit/hs3/test/testHS3AuxiliaryMeasurements.cxx b/roofit/hs3/test/testHS3AuxiliaryMeasurements.cxx new file mode 100644 index 0000000000000..e25c2e0c85658 --- /dev/null +++ b/roofit/hs3/test/testHS3AuxiliaryMeasurements.cxx @@ -0,0 +1,1391 @@ +// Tests for the auxiliary measurements (constraint terms) of HistFactory models in HS3 +// Authors: Carsten D. Burgard, DESY/CMS, 11/2026 + +// A classical HistFactory workspace is written to HS3 in every layout of the auxiliary measurements that the flags +// RooJSONFactoryWSTool::Config::exportArrayDist and ::exportConstraintData select, and read back. The model must stay +// the same: the same likelihood, the same toys, and the same document when it is written again, also if the layout +// changes between the exports. + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using RooFit::Detail::JSONNode; +using RooFit::Detail::JSONTree; + +namespace { + +// The four combinations of the two export flags. The HS3 documentation shows three layouts for auxiliary +// measurements: the terms in "aux_distributions" (chapter Likelihoods), each constraint paired with its own data set, +// and one array distribution paired with one unbinned data set (both in chapter Auxiliary measurements). +struct Style { + const char *name; + bool arrays; // config().exportArrayDist + bool constraintData; // config().exportConstraintData +}; + +const std::vector