diff --git a/README/ReleaseNotes/v642/index.md b/README/ReleaseNotes/v642/index.md index 5f2747d8ce057..a1ccbb1062717 100644 --- a/README/ReleaseNotes/v642/index.md +++ b/README/ReleaseNotes/v642/index.md @@ -89,7 +89,7 @@ The following people have contributed to this new version: * The overloads of `RooAbsReal::createChi2()` and `RooAbsReal::chi2FitTo()` that take unbinned **RooDataSet** data objects were deprecated in ROOT 6.40 and are now removed. * The **RooStats::HybridPlot** class and the related **HybridResult::GetPlot** method were deprecated in ROOT 6.40 and are now removed. * The JSON I/O of RooFitHS3 through "export keys" and "factory expressions", which mapped the proxies and constructor arguments of a class to JSON keys, is removed, together with the functions `RooFit::JSONIO::loadExportKeys()`, `loadFactoryExpressions()`, `clearExportKeys()`, `clearFactoryExpressions()`, `printExportKeys()`, `printFactoryExpressions()`, `exportKeys()` and `importExpressions()`. The classes that were covered this way now have C++ importers and exporters, and the JSON that RooFit reads and writes is unchanged. To add JSON I/O for your own class, implement a `RooFit::JSONIO::Importer` and `RooFit::JSONIO::Exporter` instead, as explained in the [RooFitHS3 developer documentation](https://root.cern/doc/v644/group__roofit__dev__docs__hs3.html). -* Some helper functions of `RooJSONFactoryWSTool` and `RooFit::Detail::JSONNode` that were only meant for the importers and exporters in ROOT are removed: `RooJSONFactoryWSTool::createNewJSONTree()`, `makeVariablesNode()`, `getRooFitInternal()` and `exportAxis()`, as well as `JSONNode::has_key()`, `JSONNode::operator>>()`, the `JSONNode::fill_seq()` overloads with a maximum size or a transformation function, the decrement and arrow operators of the child iterators, and the operators that stream a map into a node, the children of a non-const node into a `std::vector`, or a node into a `std::ostream` (use `JSONNode::writeJSON()` for the latter). An exporter for your own class can still write the axes and bin contents of a histogram with `RooJSONFactoryWSTool::exportHisto()`, and an array of numbers with `exportArray()`. +* Some helper functions of `RooJSONFactoryWSTool` and `RooFit::Detail::JSONNode` that were only meant for the importers and exporters in ROOT are removed: `RooJSONFactoryWSTool::createNewJSONTree()`, `makeVariablesNode()`, `getRooFitInternal()` and `exportAxis()`, the struct `RooJSONFactoryWSTool::CombinedData`, as well as `JSONNode::has_key()`, `JSONNode::operator>>()`, the `JSONNode::fill_seq()` overloads with a maximum size or a transformation function, the decrement and arrow operators of the child iterators, and the operators that stream a map into a node, the children of a node into a `std::vector` (stream the node itself instead), or a node into a `std::ostream` (use `JSONNode::writeJSON()` instead). An exporter for your own class can still write the axes and bin contents of a histogram with `RooJSONFactoryWSTool::exportHisto()`, and an array of numbers with `exportArray()`. * The conversion from Python set to **RooArgSet** is deprecated and won't work anymore in ROOT 6.44. The problem is that Python sets are unordered while RooArgSets are ordered, and this mismatch can lead to subtle problems later on. Prefer conversion from Python lists or tuples, which are ordered too. * Flattening nested **RooSimultaneous** in the constructor is deprecated and will be removed in ROOT 6.44. It double counts extended components that don't depend on all index categories ([GitHub issue #23342](https://github.com/root-project/root/issues/23342)) and breaks nested RooSimultaneous whose index category is a parameter ("switch mode"). From ROOT 6.44, nested RooSimultaneous are only supported in switch mode. To combine channels, use one RooSimultaneous over a **RooCategory** with one state per channel. Workspaces read from files are not affected. * The `RooTrace` class is deprecated and will be removed in ROOT 6.44. It was a RooFit-specific memory tracer whose instrumentation hooks are compiled out by default, so it has been inert and untested for years. For memory debugging, please use general-purpose tools such as AddressSanitizer or Valgrind instead. diff --git a/roofit/hs3/CMakeLists.txt b/roofit/hs3/CMakeLists.txt index ab3af61ff88ac..6a73401bb560e 100644 --- a/roofit/hs3/CMakeLists.txt +++ b/roofit/hs3/CMakeLists.txt @@ -16,6 +16,7 @@ add_library(hs3cpp OBJECT src/HS3Parser.cxx src/HS3Upgrade.cxx src/HS3Validate.cxx + src/HS3Variates.cxx src/HS3Writer.cxx src/JSONInterface.cxx ) @@ -63,7 +64,6 @@ if(NOT CMAKE_VERSION VERSION_LESS "3.23.0") src/HS3Schema.h src/JSONIOUtils.h src/JSONReadRecorder.h - src/static_execute.h ) endif() diff --git a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h index b9dfb0a913be6..556b56e710dec 100644 --- a/roofit/hs3/inc/RooFit/Detail/JSONInterface.h +++ b/roofit/hs3/inc/RooFit/Detail/JSONInterface.h @@ -14,8 +14,6 @@ #define RooFit_Detail_JSONInterface_h #include -#include -#include #include #include #include @@ -37,13 +35,7 @@ class JSONNode { ++_pos; return *this; } - child_iterator_t &operator--() - { - --_pos; - return *this; - } Nd &operator*() const { return **_pos; } - Nd &operator->() const { return **_pos; } friend bool operator!=(child_iterator_t const &lhs, child_iterator_t const &rhs) { return lhs._pos != rhs._pos; } friend bool operator==(child_iterator_t const &lhs, child_iterator_t const &rhs) { return lhs._pos == rhs._pos; } @@ -70,16 +62,13 @@ class JSONNode { JSONNode(JSONNode const &) = delete; JSONNode &operator=(JSONNode const &) = delete; -public: void writeJSON(std::ostream &os) const; -public: JSONNode &operator<<(std::string const &s); inline JSONNode &operator<<(const char *s) { return *this << std::string(s); } JSONNode &operator<<(int i); JSONNode &operator<<(double d); JSONNode &operator<<(bool b); - const JSONNode &operator>>(std::string &v) const; JSONNode &operator[](std::string const &k); const JSONNode &operator[](std::string const &k) const; bool is_container() const; @@ -103,7 +92,6 @@ class JSONNode { std::optional try_val_bool() const; template T val_t() const; - bool has_key() const; bool has_val() const; bool has_child(std::string const &) const; JSONNode &append_child(); @@ -126,28 +114,6 @@ class JSONNode { } } - template - void fill_seq(Collection const &coll, size_t nmax) - { - set_seq(); - size_t n = 0; - for (auto const &item : coll) { - if (n >= nmax) - break; - append_child() << item; - ++n; - } - } - - template - void fill_seq(Collection const &coll, TransformationFunc func) - { - set_seq(); - for (auto const &item : coll) { - append_child() << func(item); - } - } - template void fill_mat(Matrix const &mat) { @@ -161,32 +127,22 @@ class JSONNode { } } - JSONNode const *find(std::string const &key) const - { - auto &n = *this; - return n.has_child(key) ? &n[key] : nullptr; - } + /// The component at the end of a path of keys, or nullptr if a component on the path doesn't exist. + JSONNode const *find(std::string const &key) const { return has_child(key) ? &(*this)[key] : nullptr; } template JSONNode const *find(std::string const &key, Keys_t const &...keys) const { - auto &n = *this; - return n.has_child(key) ? n[key].find(keys...) : nullptr; + return has_child(key) ? (*this)[key].find(keys...) : nullptr; } - JSONNode &get(std::string const &key) - { - auto &n = *this; - n.set_map(); - return n[key]; - } + /// The component at the end of a path of keys. The nodes on the path become maps. + JSONNode &get(std::string const &key) { return set_map()[key]; } template JSONNode &get(std::string const &key, Keys_t const &...keys) { - auto &next = get(key); - next.set_map(); - return next.get(keys...); + return get(key).get(keys...); } private: @@ -219,55 +175,16 @@ class JSONTree { std::unique_ptr _storage; }; -std::ostream &operator<<(std::ostream &os, RooFit::Detail::JSONNode const &s); - -template -std::vector &operator<<(std::vector &v, RooFit::Detail::JSONNode::children_view const &cv) -{ - for (const auto &e : cv) { - v.push_back(e.val_t()); - } - return v; -} - -template -std::vector &operator<<(std::vector &v, RooFit::Detail::JSONNode::const_children_view const &cv) -{ - for (const auto &e : cv) { - v.push_back(e.val_t()); - } - return v; -} - template std::vector &operator<<(std::vector &v, RooFit::Detail::JSONNode const &n) { if (!n.is_seq()) { throw std::runtime_error("node " + n.key() + " is not of sequence type!"); } - v << n.children(); - return v; -} - -template -RooFit::Detail::JSONNode & -operator<<(RooFit::Detail::JSONNode &n, const std::unordered_map &m) -{ - n.set_map(); - for (const auto &it : m) { - n[it.first] << it.second; - } - return n; -} - -template -RooFit::Detail::JSONNode &operator<<(RooFit::Detail::JSONNode &n, const std::map &m) -{ - n.set_map(); - for (const auto &it : m) { - n[it.first] << it.second; + for (const auto &e : n.children()) { + v.push_back(e.val_t()); } - return n; + return v; } template <> diff --git a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h index 291be7fc4a7cf..aac0e3329f720 100644 --- a/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h +++ b/roofit/hs3/inc/RooFitHS3/RooJSONFactoryWSTool.h @@ -35,11 +35,14 @@ struct ParameterPoint; struct ParameterValue; struct ToolExportState; struct ToolImportState; +struct Version; template struct Ref; using DistributionRef = Ref; void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, std::vector channelConstraints); +std::string const *importedArrayComponentForm(RooJSONFactoryWSTool &tool, std::string const &name); +void recordImportedArrayComponent(RooJSONFactoryWSTool &tool, std::string const &name, std::string form); } // namespace HS3 } // namespace RooFit namespace RooStats { @@ -47,7 +50,6 @@ class ModelConfig; } class RooRealVar; class RooAbsCategory; -class RooCategory; class RooSimultaneous; class RooJSONFactoryWSTool { @@ -76,12 +78,24 @@ class RooJSONFactoryWSTool { /// the JSON. bool allowExportSharedVariates = false; // Write array distributions (HS3 `array_dist`). If false, no `array_dist` is written: HistFactory constraints - // are named individually, and a RooProdPdf tagged as array distribution is written as `product_dist`. + // are named individually, and a RooProdPdf tagged as array distribution is written as `product_dist`. The + // RooProdPdf that RooFit created itself when it read an `array_dist` is only written if another object uses + // it; otherwise its components are written on their own. bool exportArrayDist = true; // How the constraint terms of HistFactory models enter the likelihood: listed in "aux_distributions", with - // the observed values as constant parameters (false), or listed in "distributions", each paired with a data set - // holding the observed values of its observables (true). + // the observed values as constant parameters (false), or paired with a data set holding the observed values + // of their observables (true). An array of constraints of a model that is a RooSimultaneous is then a component + // of the combined distribution, and its data set a component of the combined data set, so that the combined + // distribution is the sampling distribution of all measurements. Other constraints are terms of the likelihood. bool exportConstraintData = false; + /// EXPERIMENTAL: how auxiliary measurements that are components of a combined distribution (a constraint term + /// paired with the data set of its observed values, as RooFit writes them with `exportConstraintData`) are + /// read. If false, they become constraint terms of the model as in HistFactory, with global observables set + /// to the observed values; RooFit recognizes the components that it wrote like this by a mark in the "misc" + /// section of the document. If true, or without that mark, they stay components of the RooSimultaneous, with + /// their data set of one entry in the combined data set, so that the RooSimultaneous is the sampling + /// distribution of all measurements. This option and the form of the resulting workspace may still change. + bool importConstraintsAsChannels = false; }; static Config &config(); @@ -89,20 +103,14 @@ class RooJSONFactoryWSTool { static RooWorkspace sanitizeWS(const RooWorkspace &ws); static RooWorkspace cleanWS(const RooWorkspace &ws, bool onlyModelConfig = false); - struct CombinedData { - std::string name; - std::vector columns; - }; - RooJSONFactoryWSTool(RooWorkspace &ws); ~RooJSONFactoryWSTool(); static std::string name(const RooFit::Detail::JSONNode &n); static bool isValidName(const std::string &str); - static bool testValidName(const std::string &str, bool forcError); + static bool testValidName(const std::string &name, bool forceError); static std::string sanitizeName(const std::string str); - static void rebuildModelConfigInWorkspace(RooStats::ModelConfig *mc, RooWorkspace &ws); static RooFit::Detail::JSONNode &appendNamedChild(RooFit::Detail::JSONNode &node, std::string const &name); static RooFit::Detail::JSONNode const *findNamedChild(RooFit::Detail::JSONNode const &node, std::string const &name); @@ -191,22 +199,18 @@ class RooJSONFactoryWSTool { readBinnedData(const RooFit::Detail::JSONNode &n, const std::string &namecomp, RooArgSet const &vars); bool importJSON(std::string const &filename); - bool importJSON(std::istream &os); + bool importJSON(std::istream &is); bool exportJSON(std::string const &fileName); bool exportJSON(std::ostream &os); std::string exportJSONtoString(); bool importJSONfromString(const std::string &s); void importJSONElement(const std::string &name, const std::string &jsonString); - void importVariableElement(const RooFit::Detail::JSONNode &n); + void importVariableElement(const RooFit::Detail::JSONNode &elementNode); void importFunction(const RooFit::Detail::JSONNode &p, bool importAllDependants); void importFunction(const std::string &jsonString, bool importAllDependants); - static std::unique_ptr createNewJSONTree(); - - static RooFit::Detail::JSONNode &makeVariablesNode(RooFit::Detail::JSONNode &rootNode); - // error handling helpers class DependencyMissingError : public std::runtime_error { std::string _parent, _child, _class; @@ -224,15 +228,6 @@ class RooJSONFactoryWSTool { const std::string &classname() const { return _class; } }; - template - static RooFit::Detail::JSONNode &getRooFitInternal(RooFit::Detail::JSONNode &node, Keys_t const &...keys) - { - return node.get("misc", "ROOT_internal", keys...); - } - - static void exportAxis(RooFit::Detail::JSONNode &obsNode, RooRealVar const &var); - static void exportCategoricalAxis(RooFit::Detail::JSONNode &obsNode, RooCategory const &cat); - static void exportHisto(RooArgSet const &vars, std::size_t n, double const *contents, RooFit::Detail::JSONNode &output); @@ -260,8 +255,19 @@ class RooJSONFactoryWSTool { std::string getStringAttribute(const std::string &obj, const std::string &attrib); void setStringAttribute(const std::string &obj, const std::string &attrib, const std::string &value); - /// The "hs3_version" of the document that is currently being imported, or an empty string outside of an import. - std::string const &hs3VersionInput() const { return _hs3VersionInput; } + /// The HS3 version of the document that is currently being imported. Outside of an import it is 0.0.0, like a + /// version that can't be parsed. + RooFit::HS3::Version hs3Version() const; + + /// The constraint terms that the document that is currently being imported adds to the HistFactory distribution + /// `dist`, besides those that its modifiers name, or nullptr if there are none. + std::vector const *channelConstraints(std::string const &dist) const; + + /// Whether the constraint term `pdf` contributes to the likelihoods of the document that is currently being + /// imported as a component of a combined distribution with data, by itself or as a component of an array + /// distribution. It is then a measurement of its own, and must not be multiplied into the distributions that + /// name it as a constraint as well. + bool isConstraintReadAsChannel(RooAbsPdf const &pdf) const; private: template @@ -282,12 +288,12 @@ class RooJSONFactoryWSTool { } } void exportArrayDists(); - bool exportConstraintData(std::string const &term, RooArgSet const *globalObservables); + bool exportConstraintData(std::string const &term, RooArgSet const *globalObservables, std::string const &dataName); void exportSimultaneous(RooSimultaneous const &simPdf); void recordCategory(RooAbsCategory const &cat); void exportData(RooAbsData const &data); - RooJSONFactoryWSTool::CombinedData exportCombinedData(RooAbsData const &data); + void exportCombinedData(RooAbsData const &data); void importAllNodes(RooFit::Detail::JSONNode &n); void importDistribution(RooFit::HS3::Distribution const &distribution); @@ -296,23 +302,23 @@ class RooJSONFactoryWSTool { void warnIfFreeWithoutDomain(RooRealVar const &var); void importDependants(const RooFit::Detail::JSONNode &n); - void exportVariable(const RooAbsArg *v, RooFit::HS3::ParameterPoint &point, bool storeConstant); - void exportVariables(const RooArgSet &allElems, RooFit::HS3::ParameterPoint &point, bool storeConstant); + void exportVariable(const RooAbsArg *v, RooFit::HS3::ParameterPoint &point); void exportAllObjects(RooFit::Detail::JSONNode &n); - void exportModelConfig(RooStats::ModelConfig const &mc, - const std::vector &combined, - const std::vector &single); + void exportModelConfig(RooStats::ModelConfig const &mc, const std::vector &singleDataSets); void exportSingleModelConfig(RooStats::ModelConfig const &mc, std::string const &analysisName, std::string const *dataName); friend void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, RooFit::HS3::HistFactoryDist dist, std::vector channelConstraints); + friend std::string const * + RooFit::HS3::importedArrayComponentForm(RooJSONFactoryWSTool &tool, std::string const &name); + friend void + RooFit::HS3::recordImportedArrayComponent(RooJSONFactoryWSTool &tool, std::string const &name, std::string form); // member variables - std::string _hs3VersionInput; RooWorkspace &_workspace; // objects to represent intermediate information diff --git a/roofit/hs3/src/HS3Builder.h b/roofit/hs3/src/HS3Builder.h index dda8410ddc2a9..52a9b5db2ec22 100644 --- a/roofit/hs3/src/HS3Builder.h +++ b/roofit/hs3/src/HS3Builder.h @@ -22,6 +22,7 @@ #include class RooAbsArg; +class RooAbsBinning; class RooDataHist; class RooJSONFactoryWSTool; class RooRealVar; @@ -33,14 +34,16 @@ class Importer; namespace HS3 { -/// Forward the warnings to the message service, and throw if there are errors. -void report(Diagnostics const &diagnostics); +/// Forward the warnings to the message service, and throw if there are errors. The paths of the diagnostics start with +/// the name of the object, if they are about an object that doesn't come with a document. +void report(Diagnostics const &diagnostics, std::string const &object = ""); /// Create the variables for the axes of binned data. RooArgSet createObservables(std::vector const &axes); -/// The axis of binned data for a variable. +/// The axis of binned data for a variable, or for a binning of a variable with the given name. BinnedAxis toBinnedAxis(RooRealVar const &var); +BinnedAxis toBinnedAxis(std::string const &name, RooAbsBinning const &binning); /// The name of an object, or the value of a literal constant. NameOrNumber toNameOrNumber(RooAbsArg const &arg); @@ -49,23 +52,29 @@ NameOrNumber toNameOrNumber(RooAbsArg const &arg); std::unique_ptr createDataHist(std::string const &name, RooArgSet const &vars, std::vector const &contents, std::vector const *errors); -/// Create the RooFit objects of a HistFactory distribution in the workspace of the tool. The constraint terms that no -/// modifier names are passed separately. -void buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist, - std::vector const *channelConstraints = nullptr); +/// Create the RooFit objects of a HistFactory distribution in the workspace of the tool. +void buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist); + +/// Create the RooSimultaneous of a combined distribution in the workspace of the tool. The other terms of the +/// likelihoods that have the distribution are the candidates for the constraint terms of its components. +void buildCombinedDist(RooJSONFactoryWSTool &tool, std::string const &name, CombinedDist const &dist, + std::vector const &otherLikelihoodTerms = {}); /// Called by the exporter of a HistFactory distribution, which is then written by the tool. The constraint terms that /// no modifier names are passed separately. void setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, std::vector channelConstraints); +/// The canonical form (see arrayComponentForm()) of the component of an array distribution that this import created +/// under the name `name`, or nullptr if it created none of that name. +std::string const *importedArrayComponentForm(RooJSONFactoryWSTool &tool, std::string const &name); + +/// Record that this import created the component `name` of an array distribution, with the canonical form `form`. +void recordImportedArrayComponent(RooJSONFactoryWSTool &tool, std::string const &name, std::string form); + /// Whether this is the importer that RooFit registers for HistFactory distributions, as opposed to the one of a user. bool isBuiltinHistFactoryImporter(JSONIO::Importer const &importer); -/// What the built-in importer does with a HistFactory distribution, plus the constraint terms of the channel. -void importHistFactoryNode(RooJSONFactoryWSTool &tool, Detail::JSONNode const &node, - std::vector const *channelConstraints); - } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Document.cxx b/roofit/hs3/src/HS3Document.cxx index bb3498a84a6cd..b4f658d5e370d 100644 --- a/roofit/hs3/src/HS3Document.cxx +++ b/roofit/hs3/src/HS3Document.cxx @@ -51,13 +51,6 @@ DomainAxis const *DomainRanges::find(std::string const &name) const return found != _ranges.end() ? found->second : nullptr; } -namespace { - -std::string const &nameOf(Modifier const &modifier) -{ - return std::visit([](auto const &m) -> std::string const & { return m.name; }, modifier); -} - char const *typeOf(Modifier const &modifier) { return std::visit( @@ -67,6 +60,8 @@ char const *typeOf(Modifier const &modifier) modifier); } +namespace { + // What duplicates must have in common to be combined. template char const *differences(Modifier_t const &l, Modifier_t const &r) diff --git a/roofit/hs3/src/HS3Document.h b/roofit/hs3/src/HS3Document.h index c4a9c120a23fb..6f69e245a2795 100644 --- a/roofit/hs3/src/HS3Document.h +++ b/roofit/hs3/src/HS3Document.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -41,13 +42,7 @@ struct Version { inline bool operator<(Version const &l, Version const &r) { - if (l.major != r.major) { - return l.major < r.major; - } - if (l.minor != r.minor) { - return l.minor < r.minor; - } - return l.patch < r.patch; + return std::tie(l.major, l.minor, l.patch) < std::tie(r.major, r.minor, r.patch); } /// Combines lambdas into one visitor for std::visit(). The compiler complains about the alternatives without lambda. @@ -58,6 +53,33 @@ struct overloaded : Ts... { template overloaded(Ts...) -> overloaded; +/// The name of the alternative that a variant holds, e.g. of an axis or of a modifier. +template +std::string const &nameOf(std::variant const &v) +{ + return std::visit([](auto const &alternative) -> std::string const & { return alternative.name; }, v); +} + +/// The object with the name `name` among the objects of a section, e.g. `findByName(doc.data, "obsData")`, or nullptr. +template +auto findByName(Objects_t &objects, std::string const &name) -> decltype(&*objects.begin()) +{ + for (auto &object : objects) { + if (object.name == name) { + return &object; + } + } + return nullptr; +} + +/// The body of the object `name` if it is a `Body_t`, e.g. `find(doc.data, "obsData")`, or nullptr. +template +auto find(Objects_t &objects, std::string const &name) -> decltype(std::get_if(&objects.begin()->body)) +{ + auto *object = findByName(objects, name); + return object ? std::get_if(&object->body) : nullptr; +} + /// A name that says what it refers to, so that for example a DataRef can't be used as a DomainRef. template struct Ref { @@ -108,6 +130,9 @@ struct ContinuousAxis { std::string name; std::optional min; std::optional max; + /// Whether the axis has a fixed "value", which is only allowed in domains. Older versions of RooFit wrote it for + /// the constant observables of data sets. The value is not used. + bool hasFixedValue = false; }; /// The axis of binned data. @@ -188,32 +213,62 @@ struct BinnedData { std::optional uncertainty; }; -/// The sub-data set of a combined data set for one key, which has one label per column. -struct CombinedDataComponent { +/// The data set or distribution that a combined data set or distribution selects for one key, which has one label per +/// column. +template +struct CombinedComponent { std::vector key; - DataRef data; + Ref_t ref; }; -/// Data sets that are selected by a key (HS3 #119). The columns are categorical axes of the domains. -struct CombinedData { +/// Data sets or distributions that are selected by a key (HS3 #119). The columns are categorical axes of the domains. +template +struct Combined { std::vector columns; - std::vector components; + std::vector> components; + + /// The component for a key, or nullptr. + CombinedComponent const *find(std::vector const &key) const + { + for (auto const &component : components) { + if (component.key == key) { + return &component; + } + } + return nullptr; + } }; +using CombinedData = Combined; +using CombinedDist = Combined; + struct Data { std::string name; std::variant body; + + /// The number of factors that a distribution paired with the data contributes to a likelihood: the number of + /// entries of unbinned data, and one for point data and binned data. Combined data gives one as well, although + /// it is its components that are paired with distributions. + std::size_t numEntries() const + { + auto const *unbinned = std::get_if(&body); + return unbinned ? unbinned->numEntries() : 1; + } }; +/// The data of a likelihood term: a data set, a number given in-line, or none (std::monostate) in a pure model. struct LikelihoodTerm { DistributionRef distribution; - std::variant data; + std::variant data; }; struct Likelihood { std::string name; std::vector terms; std::vector auxDistributions; + /// Whether "data" is null: the likelihood only groups the distributions of a model without data (e.g. for toy + /// generation), and it can't be evaluated before a data set is created. + bool pureModel = false; }; struct Analysis { @@ -229,6 +284,8 @@ struct Analysis { /// its type in RooFit::JSONIO::importers(). struct GenericObject { std::string type; + /// The object in the JSON tree that the document was read from. In a document that is exported, the distributions + /// refer to what the exporters wrote once all of them are written, and the functions don't have a node. Detail::JSONNode const *node = nullptr; }; @@ -261,13 +318,7 @@ inline bool operator!=(Interpolation const &l, Interpolation const &r) } inline bool operator<(Interpolation const &l, Interpolation const &r) { - if (l.type != r.type) { - return l.type < r.type; - } - if (l.in != r.in) { - return l.in < r.in; - } - return l.out < r.out; + return std::tie(l.type, l.in, l.out) < std::tie(r.type, r.in, r.out); } inline constexpr Interpolation additivePiecewiseLinear{InterpolationType::Add, InterpolationFunction::Poly1, @@ -390,6 +441,9 @@ struct CustomModifier { using Modifier = std::variant; +/// The type of a modifier, as the standard names it. +char const *typeOf(Modifier const &modifier); + struct Sample { std::string name; std::vector contents; @@ -414,7 +468,7 @@ struct HistFactoryDist { struct Distribution { std::string name; - std::variant body; + std::variant body; }; struct Function { @@ -454,6 +508,7 @@ struct Attributes { bool isConstVar = false; ///< How ROOT versions before HS3 0.3 marked named RooConstVars. }; +/// A RooSimultaneous as older versions of RooFit wrote it, before HS3 had combined distributions. It is only read. struct CombinedDistribution { std::string indexCat; std::vector labels; @@ -461,6 +516,7 @@ struct CombinedDistribution { std::vector distributions; }; +/// The data set of a RooSimultaneous as older versions of RooFit wrote it. It is only read. struct CombinedDataset { std::string indexCat; std::vector labels; @@ -490,6 +546,9 @@ struct RooFitInternal { /// The constraint terms of HistFactory distributions that no modifier names, by distribution. They are /// also listed in the "aux_distributions" of the likelihoods, which is what other implementations use. std::map> channelConstraints; + /// The components of a combined distribution that are auxiliary measurements (constraint terms paired with the + /// data set of their observed values), by combined distribution. RooFit can read them back as constraint terms. + std::map> auxiliaryChannels; /// Axes that the components of a combined data set share, but that are no variate of any distribution, e.g. an /// event number, by combined data set. They have no statistical meaning, and may be shared. std::map> bookkeepingColumns; @@ -519,9 +578,37 @@ namespace HS3 { struct ValidateOptions; std::optional validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &options); + +/// What readAuxiliaryChannelsAsConstraints() changed. +struct AuxiliaryConstraints { + std::set terms; ///< The components that are no longer components of their combined distribution. + std::set data; ///< Their data sets, which hold the observed values of global observables. + std::vector warnings; +}; + +/// RooFit writes an array of constraints with the data set of its observed values as a component of the combined +/// distribution of the model (RooJSONFactoryWSTool::Config::exportConstraintData), and marks it in +/// misc.ROOT_internal.auxiliary_channels. Unless these components are to stay components (`asChannels`), the +/// document is changed to what RooFit wrote before: the component and its data set become a term of every likelihood +/// that has the combined distribution, which is then read as a constraint term with global observables. The label of +/// the component is taken out of the index category. +/// +/// The same is done for a document that lists a distribution twice: as a component of a combined distribution whose +/// data set has no entries for it, and as a term of its own with data. It contributes once, as the term, but a +/// component without data is not a constraint term for RooFit, which would leave it out of the likelihood. Such a +/// document is read like the ones above, with a warning. If the components stay components, the data of the term +/// becomes the data of the component instead, and the term is removed. +AuxiliaryConstraints readAuxiliaryChannelsAsConstraints(ValidatedHS3Document &doc, bool asChannels); + +/// A number given in-line in the "data" of a likelihood, or a point data set without axes, is the value of the single +/// variate of the paired distribution (HS3 0.3). The numbers become point data sets named "_data", and +/// the point data sets get their axis, so that every term of a likelihood refers to a data set, or has no data. +/// Returns the errors: a distribution without a single variate, or a point data set with an axis that is not the +/// variate of its distribution. +std::vector resolvePointData(ValidatedHS3Document &doc); } // namespace HS3 -/// An HS3 document. It is move-only, because generic objects refer to the JSON tree that the document was read from. +/// An HS3 document. It is move-only, because generic objects refer to the JSON tree of the document. class HS3Document { public: HS3::Metadata metadata; @@ -545,8 +632,6 @@ namespace HS3 { /// Options for upgradeDocument() and parseDocument(). struct ParseOptions { - /// Fragments of a document can be imported without metadata. - bool requireMetadata = true; /// If false, HistFactory distributions stay generic objects, because a user registered their own importer. bool typedHistFactory = true; }; @@ -563,9 +648,15 @@ void upgradeHistFactoryDist(Detail::JSONNode &distribution, Version const &versi /// document. Problems are reported in `diagnostics`, and if there are errors the document must not be used. HS3Document parseDocument(Detail::JSONNode const &root, Diagnostics &diagnostics, ParseOptions const &options = {}); +/// Reads a distribution that doesn't come with a document. +Distribution parseDistribution(Detail::JSONNode const &distribution, Diagnostics &diagnostics); + /// Reads a HistFactory distribution that doesn't come with a document. HistFactoryDist parseHistFactoryDist(Detail::JSONNode const &distribution, Diagnostics &diagnostics); +/// Reads a data set that doesn't come with a document. +Data parseData(Detail::JSONNode const &data, Diagnostics &diagnostics); + /// Reads the axes of binned data that doesn't come with a document. std::vector parseBinnedAxes(Detail::JSONNode const &axes, Diagnostics &diagnostics); @@ -581,15 +672,21 @@ void writeDocument(Detail::JSONNode &root, HS3Document const &doc); /// Writes a HistFactory distribution, without its name. void writeHistFactoryDist(Detail::JSONNode &node, HistFactoryDist const &dist); +/// Writes a combined distribution, without its name. +void writeCombinedDist(Detail::JSONNode &node, CombinedDist const &dist); + +/// Writes the axes, contents and uncertainties of binned data, without its name and type. +void writeBinnedData(Detail::JSONNode &node, BinnedData const &data); + +/// Writes an axis of binned data. +void writeAxis(Detail::JSONNode &node, BinnedAxis const &axis); + /// Writes the functions "interpolation" and "interpolation0d", with a single interpolation if they are all the same. void writeInterpolatedFunction(Detail::JSONNode &node, InterpolatedFunction const &function); void writeInterpolatedNumber(Detail::JSONNode &node, InterpolatedNumber const &function); void writeInterpolation(Detail::JSONNode &node, Interpolation const &interpolation); -/// Writes a number, as an integer if it has no fractional part. -void writeNumber(Detail::JSONNode &node, double value); - /// Writes an array of numbers, each as an integer if it has no fractional part. void writeContents(Detail::JSONNode &node, std::vector const &values); @@ -604,7 +701,8 @@ struct ValidateOptions { } // namespace HS3 /// A document that passed validate(). Only validate() can create it, so the code that builds RooFit objects can rely on -/// the checks. +/// the checks. Afterwards, it can only be changed by the functions of this layer that adapt it to what RooFit can +/// build, which are trusted to leave it consistent. class ValidatedHS3Document { public: HS3Document const &doc() const { return _doc; } @@ -613,6 +711,8 @@ class ValidatedHS3Document { explicit ValidatedHS3Document(HS3Document doc) : _doc{std::move(doc)} {} friend std::optional HS3::validate(HS3Document doc, HS3::Diagnostics &diagnostics, HS3::ValidateOptions const &options); + friend HS3::AuxiliaryConstraints HS3::readAuxiliaryChannelsAsConstraints(ValidatedHS3Document &doc, bool asChannels); + friend std::vector HS3::resolvePointData(ValidatedHS3Document &doc); HS3Document _doc; }; @@ -627,8 +727,9 @@ validate(HS3Document doc, Diagnostics &diagnostics, ValidateOptions const &optio /// The checks of validate(), for a document that is still needed if it doesn't pass them. void checkDocument(HS3Document const &doc, Diagnostics &diagnostics, ValidateOptions const &options = {}); -/// Same as validate(), but for a single HistFactory distribution that doesn't come with a document. +/// Same as validate(), but for a single HistFactory or combined distribution that doesn't come with a document. void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &name, Diagnostics &diagnostics); +void validateCombinedDist(CombinedDist const &dist, std::string const &name, Diagnostics &diagnostics); /// Reads the leading "MAJOR.MINOR" and the optional ".PATCH" of an HS3 version string. bool parseVersion(std::string const &version, Version &out); @@ -641,6 +742,60 @@ void mergeDuplicateModifiers(HistFactoryDist &dist, std::string const &channel, /// Makes the most common interpolation of the modifiers the "default_interpolation", which they then don't repeat. void factorOutDefaultInterpolation(HistFactoryDist &dist); +// HS3 requires the terms of a likelihood, and the components of a combined distribution or data set, to have distinct +// variates. For this, the generic distributions are read as far as the standard says what they consist of (e.g. the +// factors of a product), through the nodes that the document refers to. + +/// Describes each variate that is shared although it must be distinct: by the terms of a likelihood, by the +/// components of a combined distribution that no likelihood lists (the others are checked through the likelihood), or +/// by the components of a combined data set, except for bookkeeping columns. +std::vector sharedVariates(HS3Document const &doc); + +/// In HS3 0.3 (HS3 PR #154, decision on HS3 #156), the "constraints" of a HistFactory modifier add no term to a +/// likelihood: every likelihood that contains the channel must list each named distribution exactly once, in +/// "distributions" or "aux_distributions", or as a factor of a product_dist or a component of a combined_dist reached +/// from them. One message for each named distribution of each likelihood that doesn't. For older documents, RooFit +/// adds the named constraints itself (see sharedVariates()). +/// +/// A distribution contributes one factor per entry of the data that it is paired with, so that being listed is not +/// the same as contributing: a component of a combined_dist without a component of the paired combined data set, or +/// with one that has no entries, contributes nothing. A constraint that contributes once elsewhere is then fine, but +/// since listing it in the model and again on its own is probably unintended, `warnings` gets a message. So it does +/// for every component of a combined distribution that has no data component at all. +std::vector unlistedConstraints(HS3Document const &doc, std::vector *warnings = nullptr); + +/// The distributions that are components of a combined distribution of a likelihood, with a component of the paired +/// combined data set that has entries: measurements of their own. +std::set componentsWithData(HS3Document const &doc); + +/// A canonical form of the component `component` of an array distribution with the type `componentType` and the +/// default parameters `defaults` (may be nullptr): the type and all fields after applying the defaults, sorted, with +/// numbers in a canonical representation. Two components with the same form are identical in the sense of HS3: they +/// are one factor of a union composition, also if they belong to different array distributions. +std::string arrayComponentForm(std::string const &componentType, RooFit::Detail::JSONNode const &component, + RooFit::Detail::JSONNode const *defaults); + +/// The variates of the "distributions" of a likelihood, which it pairs with data, with the columns of its combined +/// distributions. +std::set variatesOf(HS3Document const &doc, Likelihood const &likelihood); + +/// The variates of the distribution `distribution`, without the columns of a combined distribution. +std::set variatesOf(HS3Document const &doc, std::string const &distribution); + +/// The axes that several components of a combined data set have, by combined data set. +std::map> sharedAxes(HS3Document const &doc); + +/// The bookkeeping columns by combined data set: the declared ones (RooFitInternal::bookkeepingColumns), except for +/// those that are a variate of a distribution after all. They are exempt from the requirement of distinct variates. +std::map> bookkeepingColumns(HS3Document const &doc); + +/// Writes each term of the likelihoods once: a factor of a product that is a term of the likelihood as well would enter +/// it twice, so it is removed from the "factors" of the product in `root`, the JSON tree that the document is written +/// to. Only products that no other distribution uses are changed (being a component of a combined distribution is not +/// a use), only factors that are terms of every likelihood that has the product as a term, and a product whose factors +/// are all such terms is left as it is. Returns the removed factors by product. +std::map> writeLikelihoodTermsOnce(HS3Document const &doc, Detail::JSONNode &root); + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Parser.cxx b/roofit/hs3/src/HS3Parser.cxx index 543d9e9b71e7c..63df971929050 100644 --- a/roofit/hs3/src/HS3Parser.cxx +++ b/roofit/hs3/src/HS3Parser.cxx @@ -11,14 +11,12 @@ */ #include "HS3Schema.h" -#include "JSONIOUtils.h" #include #include #include #include #include -#include #include using RooFit::Detail::JSONNode; @@ -37,8 +35,8 @@ namespace { // so it has no additive-linear or multiplicative-linear poly6 variant. struct InterpolationCodes { Interpolation descriptor; - int piecewise; - int flexible; + int piecewise = 0; + int flexible = 0; }; constexpr int kUnrepresentable = -1; @@ -150,7 +148,8 @@ Reader::~Reader() { JSONNode const &node = _cursor.node(); // There is nothing to report if all components were read, and after an error it would only be noise. - if (!node.is_map() || node.num_children() == _seen.size() || _cursor.numErrors() != _nErrorsBefore) { + if (!_reportUnread || !node.is_map() || node.num_children() == _seen.size() || + _cursor.numErrors() != _nErrorsBefore) { return; } for (JSONNode const &child : node.children()) { @@ -240,7 +239,7 @@ void readBound(Reader &ar, const char *key, std::optional &out, double v if (!ar.has(key)) { return; } - Cursor c{ar.cursor().node()[key], ar.cursor(), key}; + Cursor c = ar.child(key); if (c.node().is_null()) { out = valueIfNull; return; @@ -254,7 +253,7 @@ constexpr double infinity = std::numeric_limits::infinity(); // Read bin edges, which must be at least two strictly ascending numbers. void readEdges(Reader &ar, std::string const &axisName, std::vector &edges) { - Cursor c{ar.cursor().node()["edges"], ar.cursor(), "edges"}; + Cursor c = ar.child("edges"); const std::string context = "\"edges\" of axis '" + axisName + "' must "; if (!c.node().is_seq()) { c.error(context + "be a list"); @@ -283,16 +282,17 @@ void describe(Reader &ar, Package &p) void describe(Reader &ar, DomainAxis &axis) { ar("name", axis.name); - if (ar.has(fixedValueKey)) { - // The domain of the parameter is the set with this single value: no range, binning or categories. + // A fixed axis (HS3 #131): the domain of the parameter is the set with this single value, without range, binning or + // categories. + if (ar.has("value")) { for (const char *key : {"min", "max", "nbins", "edges", "categories"}) { if (ar.has(key)) { - ar.cursor().error("the axis '" + axis.name + "' has \"" + fixedValueKey + "\" and \"" + key + + ar.cursor().error("the axis '" + axis.name + "' has \"value\" and \"" + key + "\", which exclude each other"); } } // Not read as an optional component, for which null would be the same as leaving it out. - ar(fixedValueKey, axis.fixedValue.emplace()); + ar("value", axis.fixedValue.emplace()); return; } readBound(ar, "min", axis.min, -infinity); @@ -322,6 +322,8 @@ void describe(Reader &ar, DomainAxis &axis) void describe(Reader &ar, ContinuousAxis &axis) { ar("name", axis.name); + // A fixed axis is only allowed in domains (HS3 0.3), which the validation reports. + axis.hasFixedValue = ar.has("value"); readBound(ar, "min", axis.min, -infinity); readBound(ar, "max", axis.max, infinity); } @@ -373,8 +375,10 @@ void describe(Reader &ar, CategoricalAxis &axis) return; } for (JSONNode const &label : labels.children()) { - if (label.is_container() || label.is_null() || label.is_number() || label.val().empty()) { - ar.cursor().error("the axis '" + axis.name + "': the entries of \"categories\" must be non-empty strings"); + // An empty label is allowed: RooFit uses it for a column that does not apply to a component of a combined + // distribution. + if (label.is_container() || label.is_null() || label.is_number()) { + ar.cursor().error("the axis '" + axis.name + "': the entries of \"categories\" must be strings"); return; } if (std::find(axis.categories.begin(), axis.categories.end(), label.val()) != axis.categories.end()) { @@ -385,102 +389,65 @@ void describe(Reader &ar, CategoricalAxis &axis) } } -namespace { - // The kind of an axis is determined by its components. -template -void readAxis(Cursor const &c, std::variant &out) +template +void describe(Reader &ar, std::variant &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader reader{c}; // A fixed axis is not categorical. That it has no categories is checked when it is read. - if (reader.has("categories") && !(std::is_same_v && reader.has(fixedValueKey))) { - CategoricalAxis axis; - describe(reader, axis); - out = std::move(axis); + if (ar.has("categories") && !(std::is_same_v && ar.has("value"))) { + describe(ar, out.template emplace()); } else { - ContinuousAxis_t axis; - describe(reader, axis); - out = std::move(axis); + describe(ar, out.template emplace()); } } -} // namespace - -void readValue(Cursor const &c, std::variant &out) -{ - readAxis(c, out); -} - -void readValue(Cursor const &c, std::variant &out) -{ - readAxis(c, out); -} - -void readValue(Cursor const &c, std::variant &out) -{ - readAxis(c, out); -} - void describe(Reader &ar, Domain &domain) { ar("name", domain.name); std::string type; - if (ar.has("type")) { - ar("type", type); - } + ar.optional("type", type); if (type != "product_domain") { ar.cursor().error("only domains of type \"product_domain\" are currently supported!"); return; } - ar.list("axes", domain.axes); + ar.optional("axes", domain.axes); } -void readValue(Cursor const &c, ParameterValue &p) +void describe(Reader &ar, ParameterValue &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; - ar("name", p.name); + ar("name", out.name); if (ar.has("value")) { - Cursor crsr{ar.cursor().node()["value"], ar.cursor(), "value"}; + Cursor crsr = ar.child("value"); if (crsr.node().is_number()) { - p.value = crsr.node().val_double(); + out.value = crsr.node().val_double(); } else { std::string label; readValue(crsr, label); - p.value = std::move(label); + out.value = std::move(label); } } - std::optional isConst; - ar("const", isConst); - p.isConst = isConst.value_or(false); - ar("err", p.error); - ar("relErr", p.relError); - ar("nbins", p.nbins); + ar.optional("const", out.isConst); + ar("err", out.error); + ar("relErr", out.relError); + ar("nbins", out.nbins); } void describe(Reader &ar, ParameterPoint &point) { ar("name", point.name); - ar.list("parameters", point.parameters); + ar.optional("parameters", point.parameters); } void describe(Reader &ar, PointData &data) { ar("value", data.value); ar("uncertainty", data.uncertainty); - ar.list("axes", data.axes); + ar.optional("axes", data.axes); } void describe(Reader &ar, UnbinnedData &data) { - ar.list("axes", data.axes); + ar.optional("axes", data.axes); ar("weights", data.weights); data.hasEntriesUncertainties = ar.has("entries_uncertainties"); @@ -488,7 +455,7 @@ void describe(Reader &ar, UnbinnedData &data) ar.cursor().error("does not define the required component \"entries\""); return; } - Cursor entries{ar.cursor().node()["entries"], ar.cursor(), "entries"}; + Cursor entries = ar.child("entries"); if (!entries.node().is_seq()) { entries.error("key 'entries' is not a list!"); return; @@ -543,7 +510,7 @@ void describe(Reader &ar, GaussianUncertainty &uncertainty) } ar("sigma", uncertainty.sigma); if (ar.has("correlation")) { - Cursor c{ar.cursor().node()["correlation"], ar.cursor(), "correlation"}; + Cursor c = ar.child("correlation"); // Besides a matrix, it can be the number 0 to indicate that there is no correlation. if (c.node().is_seq()) { readValue(c, uncertainty.correlation); @@ -555,12 +522,12 @@ void describe(Reader &ar, GaussianUncertainty &uncertainty) void describe(Reader &ar, BinnedData &data) { - ar.list("axes", data.axes); + ar.optional("axes", data.axes); ar("contents", data.contents); ar("uncertainty", data.uncertainty); // RooFit extension: older versions read the errors like in the samples of a HistFactory distribution. if (ar.has("errors")) { - Cursor errors{ar.cursor().node()["errors"], ar.cursor(), "errors"}; + Cursor errors = ar.child("errors"); if (!errors.node().is_seq()) { errors.error("errors are not in list form"); } else if (!data.uncertainty) { @@ -570,16 +537,13 @@ void describe(Reader &ar, BinnedData &data) } } -// A combined data set. Only the structure is checked here: the columns and keys are checked against the domains by the -// validation, and the sub-data sets when the data set is built. -void readCombinedData(Reader &ar, std::string const &name, CombinedData &data) +// A combined data set or distribution, whose components name what they select in `refKey`. Only the structure is +// checked here: the columns and keys are checked against the domains by the validation, and the components when they +// are built. +template +void readCombined(Reader &ar, std::string const &context, const char *refKey, Combined &out) { - std::string const context = "the combined data set '" + name + "'"; Cursor const &c = ar.cursor(); - if (ar.has("axes")) { - c.error(context + " must not have \"axes\": its columns are declared by the domains"); - return; - } auto nonEmptyList = [&](const char *key) -> JSONNode const * { if (!ar.has(key) || !c.node()[key].is_seq() || c.node()[key].num_children() == 0) { return nullptr; @@ -593,7 +557,7 @@ void readCombinedData(Reader &ar, std::string const &name, CombinedData &data) columns = nullptr; break; } - data.columns.push_back(column.val()); + out.columns.push_back(column.val()); } if (!columns) { c.error("\"columns\" of " + context + " must be a non-empty list of names of categorical axes"); @@ -604,61 +568,54 @@ void readCombinedData(Reader &ar, std::string const &name, CombinedData &data) c.error("\"components\" of " + context + " must be a non-empty list"); return; } - for (JSONNode const &component : components->children()) { - if (!component.is_map() || !component.has_child("key") || !component.has_child("dataset")) { - c.error("every component of " + context + " must have a \"key\" and a \"dataset\""); + for (JSONNode const &node : components->children()) { + if (!node.is_map() || !node.has_child("key") || !node.has_child(refKey)) { + c.error("every component of " + context + " must have a \"key\" and a \"" + refKey + "\""); return; } - JSONNode const &key = component["key"]; - CombinedDataComponent &out = data.components.emplace_back(); - if (key.is_seq() && key.num_children() == data.columns.size()) { + JSONNode const &key = node["key"]; + CombinedComponent &component = out.components.emplace_back(); + if (key.is_seq() && key.num_children() == out.columns.size()) { for (JSONNode const &label : key.children()) { - out.key.push_back(label.is_container() ? std::string{} : label.val()); + component.key.push_back(label.is_container() ? std::string{} : label.val()); } } else { - c.error("every component of " + context + " must have a \"key\" with " + std::to_string(data.columns.size()) + + c.error("every component of " + context + " must have a \"key\" with " + std::to_string(out.columns.size()) + " labels, one per column"); return; } - out.data.name = component["dataset"].val(); + component.ref.name = node[refKey].val(); } } -void readValue(Cursor const &c, Data &out) +void describe(Reader &ar, Data &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; ar("name", out.name); std::string type; ar("type", type); if (type == "binned") { - BinnedData body; - describe(ar, body); - out.body = std::move(body); + describe(ar, out.body.emplace()); } else if (type == "unbinned") { - UnbinnedData body; - describe(ar, body); - out.body = std::move(body); + describe(ar, out.body.emplace()); } else if (type == "point") { - PointData body; - describe(ar, body); - out.body = std::move(body); + describe(ar, out.body.emplace()); } else if (type == "combined") { - CombinedData body; - readCombinedData(ar, out.name, body); - out.body = std::move(body); + std::string const context = "the combined data set '" + out.name + "'"; + auto &combined = out.body.emplace(); + if (ar.has("axes")) { + ar.cursor().error(context + " must not have \"axes\": its columns are declared by the domains"); + } else { + readCombined(ar, context, "dataset", combined); + } } else if (ar.has("type")) { - c.error("unknown type of dataset '" + type + "'"); + ar.cursor().error("unknown type of dataset '" + type + "'"); } } void describe(Reader &ar, Likelihood &likelihood) { ar("name", likelihood.name); - ar.list("aux_distributions", likelihood.auxDistributions); + ar.optional("aux_distributions", likelihood.auxDistributions); std::vector distributions; ar("distributions", distributions); @@ -666,7 +623,15 @@ void describe(Reader &ar, Likelihood &likelihood) ar.cursor().error("does not define the required component \"data\""); return; } - Cursor data{ar.cursor().node()["data"], ar.cursor(), "data"}; + Cursor data = ar.child("data"); + // A pure model, i.e. distributions without any data, which a ModelConfig can refer to. + if (data.node().is_null()) { + likelihood.pureModel = true; + for (auto &distribution : distributions) { + likelihood.terms.push_back({std::move(distribution), std::monostate{}}); + } + return; + } if (!data.node().is_seq()) { data.error("must be an array"); return; @@ -707,13 +672,13 @@ void describe(Reader &ar, Analysis &analysis) "' is deprecated, please use \"" + key + "\" instead."); return true; }; - ar.list(isLegacy("parameters_of_interest", "parameter_of_interest") ? "parameter_of_interest" - : "parameters_of_interest", - analysis.parametersOfInterest); + ar.optional(isLegacy("parameters_of_interest", "parameter_of_interest") ? "parameter_of_interest" + : "parameters_of_interest", + analysis.parametersOfInterest); if (!isLegacy("domains", "domain")) { - ar.list("domains", analysis.domains); + ar.optional("domains", analysis.domains); } else if (ar.cursor().node()["domain"].is_seq()) { - ar.list("domain", analysis.domains); + ar.optional("domain", analysis.domains); } else { analysis.domains.emplace_back(); ar("domain", analysis.domains.back()); @@ -781,8 +746,10 @@ void readValue(Cursor const &c, Interpolation &out) c.error("must be a struct with components 'type', 'in', and 'out'"); return; } + // The reader reports the other components as unknown, unless there is an error. + Reader reader{c}; for (const char *component : {"type", "in", "out"}) { - if (!node.has_child(component)) { + if (!reader.has(component)) { c.error(std::string("does not define the required '") + component + "' component"); return; } @@ -817,13 +784,8 @@ void readValue(Cursor const &c, Interpolation &out) out.out = interpolationFunctionFromString(outNode.val()); if (!out.out) { c.error("has unknown extrapolation function '" + outNode.val() + "'"); - return; } } - Reader reader{c}; - for (const char *component : {"type", "in", "out"}) { - reader.has(component); - } } namespace { @@ -844,29 +806,33 @@ std::optional readConstraintType(Cursor const &c) return std::nullopt; } +// The parameter of a modifier with a single parameter and its constraint are arrays with one entry. +template +void readSingleEntry(Reader &ar, const char *key, T &out) +{ + Cursor c = ar.child(key); + if (!c.node().is_seq() || c.node().num_children() != 1) { + c.error("must be an array with one entry"); + return; + } + readValue(Cursor{c.node().child(0), c, std::size_t{0}}, out); +} + void readConstraint(Reader &ar, Constraint &out) { if (ar.has("constraints")) { - Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; - if (!c.node().is_seq() || c.node().num_children() != 1) { - c.error("must be an array with one entry"); - return; + std::optional pdf; + readSingleEntry(ar, "constraints", pdf); + if (pdf) { + out = ConstraintPdf{std::move(*pdf)}; } - JSONNode const &item = c.node().child(0); - if (item.is_null()) { - out = Unconstrained{}; - return; - } - ConstraintPdf pdf; - readValue(Cursor{item, c, std::size_t{0}}, pdf.pdf); - out = std::move(pdf); } else if (ar.has("constraint_name")) { ConstraintPdf pdf; ar("constraint_name", pdf.pdf); pdf.legacyName = true; out = std::move(pdf); } else if (ar.has("constraint")) { - if (auto type = readConstraintType(Cursor{ar.cursor().node()["constraint"], ar.cursor(), "constraint"})) { + if (auto type = readConstraintType(ar.child("constraint"))) { out = *type; } } @@ -875,24 +841,9 @@ void readConstraint(Reader &ar, Constraint &out) void readGammaConstraint(Reader &ar, GammaConstraint &out) { if (ar.has("constraints")) { - GammaConstraintPdfs constraints; - Cursor c{ar.cursor().node()["constraints"], ar.cursor(), "constraints"}; - if (!c.node().is_seq()) { - c.error("must be an array"); - return; - } - std::size_t i = 0; - for (JSONNode const &item : c.node().children()) { - constraints.pdfs.emplace_back(); - if (!item.is_null()) { - constraints.pdfs.back().emplace(); - readValue(Cursor{item, c, i}, *constraints.pdfs.back()); - } - ++i; - } - out = std::move(constraints); + ar("constraints", out.emplace().pdfs); } else if (ar.has("constraint")) { - Cursor c{ar.cursor().node()["constraint"], ar.cursor(), "constraint"}; + Cursor c = ar.child("constraint"); // "Const" is a RooFit extension. if (!c.node().is_container() && !c.node().is_null() && c.node().val() == "Const") { out = ConstantGammas{}; @@ -904,57 +855,35 @@ void readGammaConstraint(Reader &ar, GammaConstraint &out) } // namespace -// The standard doesn't require a name for modifiers, but RooFit uses it to name the objects it creates. -void readModifierName(Reader &ar, std::string &name) -{ - std::optional optionalName; - ar("name", optionalName); - name = optionalName.value_or(""); -} - // The parameter of a modifier with a single parameter is given in "parameters", or in "parameter" in older documents. void readParameter(Reader &ar, ParameterRef ¶meter) { - if (!ar.has("parameters")) { + if (ar.has("parameters")) { + readSingleEntry(ar, "parameters", parameter); + } else { ar("parameter", parameter); - return; - } - Cursor c{ar.cursor().node()["parameters"], ar.cursor(), "parameters"}; - if (!c.node().is_seq() || c.node().num_children() != 1) { - c.error("must be an array with one entry"); - return; } - readValue(Cursor{c.node().child(0), c, std::size_t{0}}, parameter); } +// The modifiers. The standard doesn't require them to have a name, but RooFit uses it to name the objects it creates. void describe(Reader &ar, NormFactor &m) { - readModifierName(ar, m.name); + ar.optional("name", m.name); readParameter(ar, m.parameter); readConstraint(ar, m.constraint); } -void describe(Reader &ar, NormSys &m) -{ - readModifierName(ar, m.name); - readParameter(ar, m.parameter); - ar.nested("data", [&](Reader &data) { - data("hi", m.hi); - data("lo", m.lo); - }); - readConstraint(ar, m.constraint); - ar("interpolation", m.interpolation); -} - void describe(Reader &ar, HistoSysVariation &variation) { ar("contents", variation.contents); ar("errors", variation.errors); } -void describe(Reader &ar, HistoSys &m) +// A normsys or histosys modifier. They have the same components, with numbers or histograms as variations. +template +void describeSys(Reader &ar, Modifier_t &m) { - readModifierName(ar, m.name); + ar.optional("name", m.name); readParameter(ar, m.parameter); ar.nested("data", [&](Reader &data) { data("hi", m.hi); @@ -966,22 +895,22 @@ void describe(Reader &ar, HistoSys &m) void describe(Reader &ar, ShapeSys &m) { - readModifierName(ar, m.name); + ar.optional("name", m.name); if (ar.has("parameter")) { // A single name stands for one parameter per bin, which are named after it. ar("parameter", m.name); } - ar.list("parameters", m.parameters); + ar.optional("parameters", m.parameters); if (ar.has("data")) { - ar.nested("data", [&](Reader &data) { data.list("vals", m.vals); }); + ar.nested("data", [&](Reader &data) { data.optional("vals", m.vals); }); } readGammaConstraint(ar, m.constraint); } void describe(Reader &ar, StatError &m) { - readModifierName(ar, m.name); - ar.list("parameters", m.parameters); + ar.optional("name", m.name); + ar.optional("parameters", m.parameters); readGammaConstraint(ar, m.constraint); } @@ -990,37 +919,28 @@ void describe(Reader &ar, CustomModifier &m) ar("name", m.name); } -void readValue(Cursor const &c, Modifier &out) +void describe(Reader &ar, Modifier &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; std::string type; ar("type", type); - auto read = [&](auto modifier) { - describe(ar, modifier); - out = std::move(modifier); - }; if (type == "normfactor") { - read(NormFactor{}); + describe(ar, out.emplace()); } else if (type == "normsys") { - read(NormSys{}); + describeSys(ar, out.emplace()); } else if (type == "histosys") { - read(HistoSys{}); + describeSys(ar, out.emplace()); } else if (type == "shapesys" || type == "shapefactor") { - read(ShapeSys{}); + describe(ar, out.emplace()); } else if (type == "staterror") { - read(StatError{}); + describe(ar, out.emplace()); } else if (type == "custom") { - read(CustomModifier{}); + describe(ar, out.emplace()); } else if (ar.has("type")) { std::string name; if (ar.has("name")) { ar("name", name); } - c.error("modifier '" + name + "' of unknown type '" + type + "'"); + ar.cursor().error("modifier '" + name + "' of unknown type '" + type + "'"); } } @@ -1031,19 +951,15 @@ void describe(Reader &ar, Sample &sample) data("contents", sample.contents); data("errors", sample.errors); }); - ar.list("modifiers", sample.modifiers); + ar.optional("modifiers", sample.modifiers); } void describe(Reader &ar, StatErrorConfig &config) { - std::optional relThreshold; - ar("relThreshold", relThreshold); - config.relThreshold = relThreshold.value_or(0.0); + ar.optional("relThreshold", config.relThreshold); readGammaConstraint(ar, config.constraint); // Written by upgradeDocument() for older documents. - std::optional impliedPoissonErrors; - ar("implied_poisson_errors", impliedPoissonErrors); - config.impliedPoissonErrors = impliedPoissonErrors.value_or(false); + ar.optional("implied_poisson_errors", config.impliedPoissonErrors); } void describe(Reader &ar, HistFactoryDist &dist) @@ -1061,7 +977,7 @@ void describe(Reader &ar, HistFactoryDist &dist) ar("default_interpolation", dist.defaultInterpolation); ar("samples", dist.samples); if (ar.has("staterror")) { - readValue(Cursor{ar.cursor().node()["staterror"], ar.cursor(), "staterror"}, dist.statError); + readValue(ar.child("staterror"), dist.statError); } } @@ -1078,49 +994,39 @@ void readDistribution(Cursor const &c, Distribution &out, ParseOptions const &op std::string type; ar("type", type); if (options.typedHistFactory && type == "histfactory_dist") { - HistFactoryDist body; - describe(ar, body); - out.body = std::move(body); + describe(ar, out.body.emplace()); + } else if (type == "combined_dist") { + std::string const context = "the combined_dist '" + out.name + "'"; + auto &combined = out.body.emplace(); + if (ar.has("extended")) { + c.error(context + " must not have \"extended\": whether it is extended follows from its components"); + } else { + readCombined(ar, context, "dist", combined); + } } else { out.body = GenericObject{type, &c.node()}; // The importers read the other components. - for (JSONNode const &child : c.node().children()) { - ar.has(child.key().c_str()); - } + ar.ignoreUnread(); } } } // namespace -void readValue(Cursor const &c, Function &out) +void describe(Reader &ar, Function &out) { - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - Reader ar{c}; ar("name", out.name); ar("type", out.body.type); - out.body.node = &c.node(); + out.body.node = &ar.cursor().node(); // The importers read the other components. - for (JSONNode const &child : c.node().children()) { - ar.has(child.key().c_str()); - } + ar.ignoreUnread(); } void describe(Reader &ar, Attributes &attributes) { if (ar.has("dict")) { - Cursor dict{ar.cursor().node()["dict"], ar.cursor(), "dict"}; - if (!dict.node().is_map()) { - dict.error("must be a struct"); - return; - } - for (JSONNode const &item : dict.node().children()) { - readValue(Cursor{item, dict, item.key().c_str()}, attributes.dict[item.key()]); - } + ar("dict", attributes.dict); } - ar.list("tags", attributes.tags); + ar.optional("tags", attributes.tags); if (ar.has("is_const_var")) { int isConstVar = 0; ar("is_const_var", isConstVar); @@ -1159,26 +1065,6 @@ void describe(Reader &ar, ModelConfigInfo &info) namespace { -template -void readMap(Reader &ar, const char *key, Map_t &out) -{ - if (!ar.has(key)) { - return; - } - Cursor c{ar.cursor().node()[key], ar.cursor(), key}; - if (c.node().is_null()) { - return; - } - if (!c.node().is_map()) { - c.error("must be a struct"); - return; - } - for (JSONNode const &item : c.node().children()) { - const std::string itemKey = item.key(); - readValue(Cursor{item, c, itemKey.c_str()}, out[itemKey]); - } -} - // The content of "misc" is not supposed to change the result of evaluating the model, so problems with the step widths // are only warnings. void readStepWidths(Cursor const &misc, std::vector &out) @@ -1219,29 +1105,28 @@ void readMisc(Cursor const &root, RooFitInternal &out) return; } Reader ar{internal}; - readMap(ar, "attributes", out.attributes); - readMap(ar, "combined_distributions", out.combinedDistributions); - readMap(ar, "combined_datasets", out.combinedDatasets); - readMap(ar, "ModelConfigs", out.modelConfigs); - readMap(ar, "histfactory_constraints", out.channelConstraints); - readMap(ar, "bookkeeping_columns", out.bookkeepingColumns); + ar.optional("attributes", out.attributes); + ar.optional("combined_distributions", out.combinedDistributions); + ar.optional("combined_datasets", out.combinedDatasets); + ar.optional("ModelConfigs", out.modelConfigs); + ar.optional("histfactory_constraints", out.channelConstraints); + ar.optional("auxiliary_channels", out.auxiliaryChannels); + ar.optional("bookkeeping_columns", out.bookkeepingColumns); } -void readMetadata(Cursor const &root, Metadata &out, ParseOptions const &options) +void readMetadata(Cursor const &root, Metadata &out) { JSONNode const *metadata = root.node().find("metadata"); JSONNode const *versionNode = metadata ? metadata->find("hs3_version") : nullptr; if (!versionNode) { - if (options.requireMetadata) { - std::stringstream ss; - ss << "The HS3 version is missing in the JSON!\n" - << "Please include the HS3 version in the metadata field, e.g.:\n" - << " \"metadata\" :\n" - << " {\n" - << " \"hs3_version\" : \"" << supportedVersion << "\"\n" - << " }"; - root.error(ss.str()); - } + std::stringstream ss; + ss << "The HS3 version is missing in the JSON!\n" + << "Please include the HS3 version in the metadata field, e.g.:\n" + << " \"metadata\" :\n" + << " {\n" + << " \"hs3_version\" : \"" << supportedVersion << "\"\n" + << " }"; + root.error(ss.str()); return; } Cursor metadataCursor{*metadata, root, "metadata"}; @@ -1266,9 +1151,9 @@ void readMetadata(Cursor const &root, Metadata &out, ParseOptions const &options Reader ar{metadataCursor}; ar.has("hs3_version"); - ar.list("packages", out.packages); - ar.list("authors", out.authors); - ar.list("publications", out.publications); + ar.optional("packages", out.packages); + ar.optional("authors", out.authors); + ar.optional("publications", out.publications); ar("description", out.description); } @@ -1278,7 +1163,7 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO { HS3Document doc; Cursor cursor{root, diagnostics}; - readMetadata(cursor, doc.metadata, options); + readMetadata(cursor, doc.metadata); if (hasErrors(diagnostics)) { return doc; } @@ -1300,12 +1185,12 @@ HS3Document parseDocument(JSONNode const &root, Diagnostics &diagnostics, ParseO } } } - ar.list("functions", doc.functions); - ar.list("data", doc.data); - ar.list("likelihoods", doc.likelihoods); - ar.list("domains", doc.domains); - ar.list("parameter_points", doc.parameterPoints); - ar.list("analyses", doc.analyses); + ar.optional("functions", doc.functions); + ar.optional("data", doc.data); + ar.optional("likelihoods", doc.likelihoods); + ar.optional("domains", doc.domains); + ar.optional("parameter_points", doc.parameterPoints); + ar.optional("analyses", doc.analyses); readMisc(cursor, doc.rooFit); return doc; } @@ -1366,7 +1251,7 @@ void readInterpolations(Reader &ar, std::size_t nParameters, InterpolationClass { out.assign(nParameters, additivePiecewiseLinear); if (ar.has("interpolations")) { - Cursor c{ar.cursor().node()["interpolations"], ar.cursor(), "interpolations"}; + Cursor c = ar.child("interpolations"); const std::size_t errorsBefore = c.numErrors(); std::vector interpolations; readValue(c, interpolations); @@ -1387,7 +1272,7 @@ void readInterpolations(Reader &ar, std::size_t nParameters, InterpolationClass if (!ar.has("interpolationCodes")) { return; } - Cursor c{ar.cursor().node()["interpolationCodes"], ar.cursor(), "interpolationCodes"}; + Cursor c = ar.child("interpolationCodes"); if (!c.node().is_seq() || c.node().num_children() != nParameters) { c.error("legacy component 'interpolationCodes' must contain one code per parameter (got " + std::to_string(c.node().is_seq() ? c.node().num_children() : 0) + " for " + std::to_string(nParameters) + @@ -1433,6 +1318,20 @@ Function_t parseInterpolated(JSONNode const &node, Diagnostics &diagnostics, Int } // namespace +Distribution parseDistribution(JSONNode const &distribution, Diagnostics &diagnostics) +{ + Distribution out; + readDistribution(Cursor{distribution, diagnostics}, out, {}); + return out; +} + +Data parseData(JSONNode const &data, Diagnostics &diagnostics) +{ + Data out; + readValue(Cursor{data, diagnostics}, out); + return out; +} + std::vector parseBinnedAxes(JSONNode const &axes, Diagnostics &diagnostics) { std::vector out; diff --git a/roofit/hs3/src/HS3Schema.h b/roofit/hs3/src/HS3Schema.h index 6dfcebed75f60..270f5cc68e3d3 100644 --- a/roofit/hs3/src/HS3Schema.h +++ b/roofit/hs3/src/HS3Schema.h @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -70,9 +69,9 @@ class Reader { template void operator()(const char *key, std::optional &field); - /// Read an array that can be omitted, which is equivalent to an empty one. + /// Read a component that can be omitted or null, which leaves the field as it is, e.g. empty for an array. template - void list(const char *key, std::vector &field); + void optional(const char *key, T &field); /// Read the components of a struct that is nested in a component. template @@ -81,20 +80,24 @@ class Reader { /// Whether the struct has a component. Checking for a component counts as reading it. bool has(const char *key); + /// Don't report the components that were not read, because they are read later by someone else. + void ignoreUnread() { _reportUnread = false; } + Cursor const &cursor() const { return _cursor; } + /// The position of a component. + Cursor child(const char *key) const { return {_cursor.node()[key], _cursor, key}; } private: Cursor const &_cursor; std::vector _seen; std::size_t _nErrorsBefore = 0; + bool _reportUnread = true; }; void readValue(Cursor const &c, std::string &out); void readValue(Cursor const &c, double &out); void readValue(Cursor const &c, int &out); void readValue(Cursor const &c, bool &out); -void readValue(Cursor const &c, ParameterValue &out); -void readValue(Cursor const &c, Data &out); void readValue(Cursor const &c, Interpolation &out); void readValue(Cursor const &c, NameOrNumber &out); @@ -107,6 +110,15 @@ void readValue(Cursor const &c, Ref &out) readValue(c, out.name); } +/// A value that can be null, like the constraint of a parameter that is not constrained. +template +void readValue(Cursor const &c, std::optional &out) +{ + if (!c.node().is_null()) { + readValue(c, out.emplace()); + } +} + template void readValue(Cursor const &c, std::vector &out) { @@ -125,6 +137,20 @@ void readValue(Cursor const &c, std::vector &out) } } +/// A struct that has a value of one type for each name. +template +void readValue(Cursor const &c, std::map &out) +{ + if (!c.node().is_map()) { + c.error("must be a struct"); + return; + } + for (Detail::JSONNode const &item : c.node().children()) { + const std::string key = item.key(); + readValue(Cursor{item, c, key.c_str()}, out[key]); + } +} + /// Everything that has a describe() function is read from a struct. template auto readValue(Cursor const &c, T &out) -> std::void_t(), out))> @@ -144,34 +170,23 @@ void Reader::operator()(const char *key, T &field) _cursor.error(std::string("does not define the required component \"") + key + "\""); return; } - readValue(Cursor{_cursor.node()[key], _cursor, key}, field); + readValue(child(key), field); } template void Reader::operator()(const char *key, std::optional &field) { - if (!has(key)) { - return; + if (has(key)) { + readValue(child(key), field); } - Detail::JSONNode const &node = _cursor.node()[key]; - if (node.is_null()) { - return; - } - field.emplace(); - readValue(Cursor{node, _cursor, key}, *field); } template -void Reader::list(const char *key, std::vector &field) +void Reader::optional(const char *key, T &field) { - if (!has(key)) { - return; - } - Detail::JSONNode const &node = _cursor.node()[key]; - if (node.is_null()) { - return; + if (has(key) && !_cursor.node()[key].is_null()) { + readValue(child(key), field); } - readValue(Cursor{node, _cursor, key}, field); } template @@ -181,12 +196,12 @@ void Reader::nested(const char *key, Fn &&fn) _cursor.error(std::string("does not define the required component \"") + key + "\""); return; } - Cursor child{_cursor.node()[key], _cursor, key}; - if (!child.node().is_map()) { - child.error("must be a struct"); + Cursor c = child(key); + if (!c.node().is_map()) { + c.error("must be a struct"); return; } - Reader reader{child}; + Reader reader{c}; fn(reader); } diff --git a/roofit/hs3/src/HS3Upgrade.cxx b/roofit/hs3/src/HS3Upgrade.cxx index 5c1216b077288..e40ae3410ff88 100644 --- a/roofit/hs3/src/HS3Upgrade.cxx +++ b/roofit/hs3/src/HS3Upgrade.cxx @@ -41,6 +41,24 @@ bool isScalar(JSONNode const &node) return !node.is_container() && !node.is_null(); } +// Calls fn(element, cursor, index) for each struct in the array `key` of `parent`, if there is one. What is skipped is +// reported by the parser. +template +void forEachStruct(JSONNode &parent, const char *key, Cursor const &parentCursor, Fn &&fn) +{ + if (!parent.is_map() || !parent.has_child(key) || !parent[key].is_seq()) { + return; + } + Cursor array{parent[key], parentCursor, key}; + std::size_t index = 0; + for (JSONNode &element : parent[key].children()) { + if (element.is_map()) { + fn(element, Cursor{element, array, index}, index); + } + ++index; + } +} + bool isConstraintType(std::string const &value) { return value == "Gauss" || value == "Poisson" || value == "LogNormal"; @@ -139,23 +157,6 @@ struct ChannelState { bool warnedShapeSys = false; }; -// Before structured interpolations were introduced, both modifier classes used the integer code 4 as their implicit -// default. The meaning of the code is class-dependent. HS3 forbids implementation-specific defaults. -void upgradeMissingInterpolation(JSONNode &mod, InterpolationClass interpolationClass, Version const &version, - bool hasDefault, std::string const &context, ChannelState &state, Cursor const &c) -{ - if (mod.has_child("interpolation") || hasDefault || !(version < explicitInterpolationVersion)) { - return; - } - if (!state.warnedInterpolation) { - c.warning(context + " is not given, and the channel has no \"default_interpolation\". Using the implicit default " - "of older RooFit versions (code 4), which is deprecated."); - state.warnedInterpolation = true; - } - mod["interpolation"] << 4; - upgradeInterpolationCode(mod, interpolationClass, c); -} - // Whether "constraints" leaves the single parameter of a modifier unconstrained. bool isExplicitlyUnconstrained(JSONNode const &constraints) { @@ -220,11 +221,20 @@ void upgradeModifier(JSONNode &mod, std::size_t index, std::string const &sample state.warnedUnconstrained = true; } } + // Before structured interpolations were introduced, both modifier classes used the integer code 4 as their + // implicit default. The meaning of the code is class-dependent. HS3 forbids implementation-specific defaults. + if (!mod.has_child("interpolation") && !hasDefaultInterpolation && version < explicitInterpolationVersion) { + if (!state.warnedInterpolation) { + c.warning("interpolation for " + type + " modifier '" + name + "' in sample '" + sampleName + + "' of channel '" + state.channelName + + "' is not given, and the channel has no \"default_interpolation\". Using the implicit default " + "of older RooFit versions (code 4), which is deprecated."); + state.warnedInterpolation = true; + } + mod["interpolation"] << 4; + } const auto interpolationClass = type == "normsys" ? InterpolationClass::Flexible : InterpolationClass::Piecewise; - const std::string context = "interpolation for " + type + " modifier '" + name + "' in sample '" + sampleName + - "' of channel '" + state.channelName + "'"; upgradeInterpolationCode(mod, interpolationClass, c); - upgradeMissingInterpolation(mod, interpolationClass, version, hasDefaultInterpolation, context, state, c); } else if (type == "shapesys" || type == "shapefactor") { if (type == "shapesys" && !(version < namedConstraintVersion) && !state.warnedShapeSys) { c.warning("the modifier type \"shapesys\" is deprecated, please use \"shapefactor\" instead."); @@ -285,105 +295,63 @@ void upgradeHistFactory(JSONNode &dist, Version const &version, IsDistribution c } } - if (!dist.has_child("samples") || !dist["samples"].is_seq()) { - return; - } const bool hasDefaultInterpolation = dist.has_child("default_interpolation"); - - Cursor samples{dist["samples"], c, "samples"}; - std::size_t iSample = 0; - for (JSONNode &sample : dist["samples"].children()) { - Cursor sampleCursor{sample, samples, iSample++}; - if (!sample.is_map() || !sample.has_child("modifiers") || !sample["modifiers"].is_seq()) { - continue; - } + forEachStruct(dist, "samples", c, [&](JSONNode &sample, Cursor const &sampleCursor, std::size_t) { const std::string sampleName = sample.has_child("name") && isScalar(sample["name"]) ? sample["name"].val() : ""; - Cursor modifiers{sample["modifiers"], sampleCursor, "modifiers"}; - std::size_t iModifier = 0; - for (JSONNode &mod : sample["modifiers"].children()) { - Cursor modCursor{mod, modifiers, iModifier}; - if (mod.is_map()) { - upgradeModifier(mod, iModifier, sampleName, version, hasDefaultInterpolation, isDistribution, state, - modCursor); - } - ++iModifier; - } - } + forEachStruct(sample, "modifiers", sampleCursor, [&](JSONNode &mod, Cursor const &modCursor, std::size_t index) { + upgradeModifier(mod, index, sampleName, version, hasDefaultInterpolation, isDistribution, state, modCursor); + }); + }); } // Older versions of RooFit wrote the binning of the observables also for unbinned data, which the standard forbids. // The edges were written without "min" and "max". -void upgradeUnbinnedAxes(JSONNode &data) +void upgradeUnbinnedAxis(JSONNode &axis) { - if (!data.has_child("axes") || !data["axes"].is_seq()) { - return; - } - for (JSONNode &axis : data["axes"].children()) { - if (!axis.is_map()) { - continue; + if (axis.has_child("edges") && axis["edges"].is_seq() && axis["edges"].num_children() > 0) { + JSONNode &edges = axis["edges"]; + JSONNode const &first = edges.child(0); + JSONNode const &last = edges.child(edges.num_children() - 1); + if (!axis.has_child("min") && first.is_number()) { + axis["min"] << first.val_double(); } - if (axis.has_child("edges") && axis["edges"].is_seq() && axis["edges"].num_children() > 0) { - JSONNode &edges = axis["edges"]; - JSONNode const &first = edges.child(0); - JSONNode const &last = edges.child(edges.num_children() - 1); - if (!axis.has_child("min") && first.is_number()) { - axis["min"] << first.val_double(); - } - if (!axis.has_child("max") && last.is_number()) { - axis["max"] << last.val_double(); - } + if (!axis.has_child("max") && last.is_number()) { + axis["max"] << last.val_double(); } - axis.erase("edges"); - axis.erase("nbins"); } + axis.erase("edges"); + axis.erase("nbins"); } -void upgradeData(JSONNode &root, Version const &version) +void upgradeData(JSONNode &root, Version const &version, Cursor const &rootCursor) { - if (!root.has_child("data") || !root["data"].is_seq() || !(version < explicitInterpolationVersion)) { + if (!(version < explicitInterpolationVersion)) { return; } - for (JSONNode &data : root["data"].children()) { - if (data.is_map() && data.has_child("type") && isScalar(data["type"]) && data["type"].val() == "unbinned") { - upgradeUnbinnedAxes(data); + forEachStruct(root, "data", rootCursor, [](JSONNode &data, Cursor const &dataCursor, std::size_t) { + if (data.has_child("type") && isScalar(data["type"]) && data["type"].val() == "unbinned") { + forEachStruct(data, "axes", dataCursor, + [](JSONNode &axis, Cursor const &, std::size_t) { upgradeUnbinnedAxis(axis); }); } - } + }); } // HS3 0.3 forbids "min" and "max" alongside "edges" (HS3 #126), which documents before 0.3 had. Where they agree with // the edges, which the parser checks, they are only redundant, so they give a warning. -void warnBoundsWithEdges(JSONNode const &axes, Cursor const &axesCursor) -{ - if (!axes.is_seq()) { - return; - } - std::size_t i = 0; - for (JSONNode const &axis : axes.children()) { - Cursor axisCursor{axis, axesCursor, i++}; - if (axis.is_map() && axis.has_child("edges") && (axis.has_child("min") || axis.has_child("max"))) { - axisCursor.warning("\"min\" and \"max\" must not be given alongside \"edges\" from HS3 0.3 on, the edges " - "give the range"); - } - } -} - -void warnBoundsWithEdges(JSONNode const &root, Version const &version, Cursor const &rootCursor) +void warnBoundsWithEdges(JSONNode &root, Version const &version, Cursor const &rootCursor) { if (version < Version{0, 3, 0}) { return; } for (const char *section : {"domains", "data", "distributions"}) { - if (!root.has_child(section) || !root[section].is_seq()) { - continue; - } - Cursor sectionCursor{root[section], rootCursor, section}; - std::size_t i = 0; - for (JSONNode const &object : root[section].children()) { - Cursor objectCursor{object, sectionCursor, i++}; - if (object.is_map() && object.has_child("axes")) { - warnBoundsWithEdges(object["axes"], Cursor{object["axes"], objectCursor, "axes"}); - } - } + forEachStruct(root, section, rootCursor, [](JSONNode &object, Cursor const &objectCursor, std::size_t) { + forEachStruct(object, "axes", objectCursor, [](JSONNode &axis, Cursor const &axisCursor, std::size_t) { + if (axis.has_child("edges") && (axis.has_child("min") || axis.has_child("max"))) { + axisCursor.warning("\"min\" and \"max\" must not be given alongside \"edges\" from HS3 0.3 on, the " + "edges give the range"); + } + }); + }); } } @@ -406,35 +374,30 @@ void upgradeDocument(JSONNode &root, Diagnostics &diagnostics, ParseOptions cons return; } const Version version = documentVersion(root); - warnBoundsWithEdges(root, version, Cursor{root, diagnostics}); - upgradeData(root, version); - - if (!root.has_child("distributions") || !root["distributions"].is_seq() || !options.typedHistFactory) { + Cursor rootCursor{root, diagnostics}; + warnBoundsWithEdges(root, version, rootCursor); + upgradeData(root, version, rootCursor); + if (!options.typedHistFactory) { return; } std::set distributionNames; - for (JSONNode const &dist : root["distributions"].children()) { - if (dist.is_map() && dist.has_child("name") && isScalar(dist["name"])) { + forEachStruct(root, "distributions", rootCursor, [&](JSONNode &dist, Cursor const &, std::size_t) { + if (dist.has_child("name") && isScalar(dist["name"])) { distributionNames.insert(dist["name"].val()); } - } + }); // The version from which on "constraint" is only a type of the standard. const bool constraintCanBeName = version < explicitInterpolationVersion; IsDistribution isDistribution = [&](std::string const &name) { return constraintCanBeName && distributionNames.count(name) > 0; }; - Cursor rootCursor{root, diagnostics}; - Cursor distributions{root["distributions"], rootCursor, "distributions"}; - std::size_t i = 0; - for (JSONNode &dist : root["distributions"].children()) { - Cursor distCursor{dist, distributions, i++}; - if (dist.is_map() && dist.has_child("type") && isScalar(dist["type"]) && - dist["type"].val() == "histfactory_dist") { + forEachStruct(root, "distributions", rootCursor, [&](JSONNode &dist, Cursor const &distCursor, std::size_t) { + if (dist.has_child("type") && isScalar(dist["type"]) && dist["type"].val() == "histfactory_dist") { upgradeHistFactory(dist, version, isDistribution, distCursor); } - } + }); } void upgradeHistFactoryDist(JSONNode &distribution, Version const &version, Diagnostics &diagnostics) diff --git a/roofit/hs3/src/HS3Validate.cxx b/roofit/hs3/src/HS3Validate.cxx index b0696fe32ced9..69ff152b4ee63 100644 --- a/roofit/hs3/src/HS3Validate.cxx +++ b/roofit/hs3/src/HS3Validate.cxx @@ -50,6 +50,8 @@ class Validator { auto const &dist = _doc.distributions[i]; if (auto const *histFactory = std::get_if(&dist.body)) { checkHistFactory(*histFactory, dist.name, at("distributions", i, dist.name)); + } else if (auto const *combined = std::get_if(&dist.body)) { + checkCombined(*combined, at("distributions", i, dist.name), "the combined_dist '" + dist.name + "'"); } } for (auto const &item : _doc.rooFit.combinedDistributions) { @@ -57,6 +59,12 @@ class Validator { requireDistribution(ref.name, "misc.ROOT_internal.combined_distributions." + item.first, "distribution"); } } + for (auto const &item : _doc.rooFit.auxiliaryChannels) { + requireDistribution(item.first, "misc.ROOT_internal.auxiliary_channels", "distribution"); + for (auto const &ref : item.second) { + requireDistribution(ref.name, "misc.ROOT_internal.auxiliary_channels." + item.first, "component"); + } + } for (auto const &item : _doc.rooFit.channelConstraints) { requireDistribution(item.first, "misc.ROOT_internal.histfactory_constraints", "distribution"); for (auto const &ref : item.second) { @@ -83,6 +91,8 @@ class Validator { } void checkHistFactory(HistFactoryDist const &dist, std::string const &name, std::string const &path); + template + void checkCombined(Combined const &combined, std::string const &path, std::string const &context); private: bool isExternal(std::string const &name) const { return _options.isExternal && _options.isExternal(name); } @@ -130,6 +140,7 @@ class Validator { void checkCategoricalAxes(); void checkLikelihoods(); void checkAnalyses(); + void checkNormalizationRanges(Analysis const &analysis, std::string const &path); void checkData(); void checkCombinedData(); @@ -138,8 +149,18 @@ class Validator { ValidateOptions const &_options; std::map> _sections; ///< The sections where a name is defined. std::set _parameters; + std::map const *> _categories; ///< Of the categorical axes of the domains. }; +// The names that the HS3 expression language reserves for itself: its literals and functions (HS3 0.3, "Identifiers"). +static bool isReservedName(std::string const &name) +{ + static const std::set reserved{"PI", "EULER", "TRUE", "FALSE", "ifelse", "exp", "log", + "sqrt", "abs", "floor", "ceil", "pow", "min", "max", + "sin", "cos", "tan", "asin", "acos", "atan", "erf"}; + return reserved.count(name) > 0; +} + // HS3 requires the names of all objects to be unique within a document, including the parameters. RooFit writes such // documents, e.g. for a range and a pdf of one name, so only ambiguous clashes in current documents are errors. void Validator::collectNames() @@ -184,7 +205,7 @@ void Validator::collectNames() // HistFactory modifiers, which are created on the fly. for (auto const &domain : _doc.domains) { for (auto const &axis : domain.axes) { - addParameter(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); + addParameter(nameOf(axis)); } } for (auto const &point : _doc.parameterPoints) { @@ -220,6 +241,22 @@ void Validator::collectNames() } } } + // Objects that expressions can refer to must not carry the reserved names of the expression language. + auto checkNotReserved = [&](char const *what, std::string const &name) { + if (isReservedName(name)) { + strictError("", "the " + std::string(what) + " '" + name + + "' has a name that the HS3 expression language reserves for itself"); + } + }; + for (auto const &dist : _doc.distributions) { + checkNotReserved("distribution", dist.name); + } + for (auto const &func : _doc.functions) { + checkNotReserved("function", func.name); + } + for (std::string const ¶meter : _parameters) { + checkNotReserved("parameter", parameter); + } for (std::string const ¶meter : _parameters) { if (char const *section = sectionOf(parameter)) { strictError("", "the name '" + parameter + "' is used in \"" + section + @@ -241,8 +278,7 @@ void Validator::checkFixedAxes() for (std::size_t i = 0; i < _doc.domains.size(); ++i) { for (auto const &axis : _doc.domains[i].axes) { // A categorical axis is passed as null. - fn(i, std::visit([](auto const &a) -> std::string const & { return a.name; }, axis), - std::get_if(&axis)); + fn(i, nameOf(axis), std::get_if(&axis)); } } }; @@ -307,6 +343,7 @@ void Validator::checkCategoricalAxes() if (!cat) { continue; } + _categories.emplace(cat->name, &cat->categories); auto inserted = categorical.insert({cat->name, Declaration{&cat->categories, &domain.name}}); if (!inserted.second && *inserted.first->second.categories != cat->categories) { error(at("domains", i, domain.name), "the categorical axis '" + cat->name + "' is listed in the domains '" + @@ -331,24 +368,49 @@ void Validator::checkCategoricalAxes() } } -// The columns of combined data sets are categorical axes of the domains, or categories that exist already, and the keys -// consist of their categories. Every sub-data set is a data set of the document that belongs to one combined data set. -void Validator::checkCombinedData() +// The columns of a combined data set or distribution are categorical axes of the domains, or categories that exist +// already, and the keys consist of their categories. +template +void Validator::checkCombined(Combined const &combined, std::string const &path, std::string const &context) { - std::map const *> categorical; - for (Domain const &domain : _doc.domains) { - for (auto const &axis : domain.axes) { - if (auto const *cat = std::get_if(&axis)) { - categorical.emplace(cat->name, &cat->categories); - } + std::vector const *> columnCategories; + std::set columns; + for (std::string const &column : combined.columns) { + auto found = _categories.find(column); + if (found == _categories.end() && !isExternal(column)) { + error(path, + "the column '" + column + "' of " + context + " is not declared by a categorical axis of a domain"); + return; } + if (!columns.insert(column).second) { + error(path, "the column '" + column + "' of " + context + " is listed twice"); + return; + } + columnCategories.push_back(found != _categories.end() ? found->second : nullptr); } - std::set combinedNames; - for (auto const &data : _doc.data) { - if (std::holds_alternative(data.body)) { - combinedNames.insert(data.name); + std::set> keys; + for (auto const &component : combined.components) { + for (std::size_t k = 0; k < component.key.size(); ++k) { + auto const *categories = columnCategories[k]; + if (categories && std::find(categories->begin(), categories->end(), component.key[k]) == categories->end()) { + error(path, "the label '" + component.key[k] + "' in position " + std::to_string(k) + " of a key of " + + context + " is not one of the categories of '" + combined.columns[k] + "'"); + } + } + if (!keys.insert(component.key).second) { + std::string key = "["; + for (std::size_t k = 0; k < component.key.size(); ++k) { + key += (k == 0 ? "\"" : ", \"") + component.key[k] + "\""; + } + error(path, "the key " + key + "] is used by two components of " + context); } } +} + +// Every sub-data set of a combined data set is a data set of the document that belongs to one combined data set. A +// component may itself be a combined data set, paired with a component that is a combined distribution. +void Validator::checkCombinedData() +{ std::set usedComponents; for (std::size_t i = 0; i < _doc.data.size(); ++i) { auto const *combined = std::get_if(&_doc.data[i].body); @@ -357,40 +419,10 @@ void Validator::checkCombinedData() } std::string const path = at("data", i, _doc.data[i].name); std::string const context = "the combined data set '" + _doc.data[i].name + "'"; - std::vector const *> columnCategories; - std::set columns; - for (std::string const &column : combined->columns) { - auto found = categorical.find(column); - if (found == categorical.end() && !isExternal(column)) { - error(path, - "the column '" + column + "' of " + context + " is not declared by a categorical axis of a domain"); - return; - } - if (!columns.insert(column).second) { - error(path, "the column '" + column + "' of " + context + " is listed twice"); - return; - } - columnCategories.push_back(found != categorical.end() ? found->second : nullptr); - } - std::set> keys; - for (CombinedDataComponent const &component : combined->components) { - for (std::size_t k = 0; k < component.key.size(); ++k) { - auto const *categories = columnCategories[k]; - if (categories && - std::find(categories->begin(), categories->end(), component.key[k]) == categories->end()) { - error(path, "the label '" + component.key[k] + "' in position " + std::to_string(k) + " of a key of " + - context + " is not one of the categories of '" + combined->columns[k] + "'"); - } - } - if (!keys.insert(component.key).second) { - std::string key = "["; - for (std::size_t k = 0; k < component.key.size(); ++k) { - key += (k == 0 ? "\"" : ", \"") + component.key[k] + "\""; - } - error(path, "the key " + key + "] is used by two components of " + context); - } - std::string const &name = component.data.name; - if (!isDefinedIn(name, "data") || combinedNames.count(name) || !usedComponents.insert(name).second) { + checkCombined(*combined, path, context); + for (auto const &component : combined->components) { + std::string const &name = component.ref.name; + if (!isDefinedIn(name, "data") || name == _doc.data[i].name || !usedComponents.insert(name).second) { error(path, "the component '" + name + "' of " + context + " is not a data set of the document, or it is already a component of another combined " "data set"); @@ -449,11 +481,89 @@ void Validator::checkAnalyses() if (analysis.prior && !isDefinedIn(analysis.prior->name, "distributions") && !isExternal(analysis.prior->name)) { strictError(path + ".prior", "the distribution '" + analysis.prior->name + "' does not exist"); } + checkNormalizationRanges(analysis, path); + } +} + +// A variate is normalized over the axis of the paired data set, or else over its axis in the domains of the analysis. +// If both exist, they must give the same range, where an absent bound is equal only to an absent bound (HS3 0.3). +void Validator::checkNormalizationRanges(Analysis const &analysis, std::string const &path) +{ + using Range = std::pair, std::optional>; + auto bound = [](std::optional value) { return value && !std::isinf(*value) ? value : std::nullopt; }; + auto format = [](Range const &range) { + auto one = [](std::optional value) { return value ? std::to_string(*value) : std::string("none"); }; + return "[" + one(range.first) + ", " + one(range.second) + "]"; + }; + + std::map domainRanges; + for (auto const &ref : analysis.domains) { + for (auto const &domain : _doc.domains) { + if (domain.name != ref.name) { + continue; + } + for (auto const &axisVariant : domain.axes) { + auto const *axis = std::get_if(&axisVariant); + if (!axis || axis->fixedValue) { + continue; // a fixed axis gives no range + } + domainRanges[axis->name] = {bound(axis->min), bound(axis->max)}; + } + } + } + if (domainRanges.empty()) { + return; + } + + auto check = [&](std::string const &dataName, std::string const &axisName, Range const &dataRange) { + auto found = domainRanges.find(axisName); + if (found != domainRanges.end() && found->second != dataRange) { + strictError(path, "the axis '" + axisName + "' of the data set '" + dataName + "' has the range " + + format(dataRange) + ", but the domains of the analysis '" + analysis.name + + "' give it the range " + format(found->second)); + } + }; + std::function checkData = [&](std::string const &dataName) { + for (auto const &data : _doc.data) { + if (data.name != dataName) { + continue; + } + // Continuous and binned axes have a range, categorical axes don't. + auto checkAxis = [&](auto const &a) { check(data.name, a.name, {bound(a.min), bound(a.max)}); }; + auto checkAxes = [&](auto const &axes) { + for (auto const &axis : axes) { + std::visit(overloaded{[](CategoricalAxis const &) {}, checkAxis}, axis); + } + }; + std::visit(overloaded{[&](PointData const &d) { std::for_each(d.axes.begin(), d.axes.end(), checkAxis); }, + [&](UnbinnedData const &d) { checkAxes(d.axes); }, + [&](BinnedData const &d) { checkAxes(d.axes); }, + [&](CombinedData const &d) { + for (auto const &component : d.components) { + checkData(component.ref.name); + } + }}, + data.body); + } + }; + for (auto const &likelihood : _doc.likelihoods) { + if (likelihood.name != analysis.likelihood.name) { + continue; + } + for (auto const &term : likelihood.terms) { + if (auto const *data = std::get_if(&term.data)) { + checkData(data->name); + } + } } } std::size_t Validator::checkAxis(BinnedAxis const &axis, std::string const &path) { + // Unlike continuous axes, binned axes always have finite bounds. + if (std::isinf(axis.min) || std::isinf(axis.max)) { + error(path, "the binned axis '" + axis.name + "' needs finite bounds"); + } if (auto const *regular = std::get_if(&axis.binning)) { if (regular->nbins <= 0) { error(path, "the axis '" + axis.name + "' needs a positive number of bins"); @@ -498,11 +608,29 @@ void Validator::checkData() { for (std::size_t i = 0; i < _doc.data.size(); ++i) { auto const &data = _doc.data[i]; + const std::string path = at("data", i, data.name); + // A fixed axis is only allowed in domains (HS3 0.3). Older versions of RooFit wrote the constant observables of + // data sets like this. + auto checkNotFixed = [&](ContinuousAxis const &axis) { + if (axis.hasFixedValue) { + strictError(path, "the axis '" + axis.name + "' of a data set must not have a fixed \"value\""); + } + }; + if (auto const *point = std::get_if(&data.body)) { + for (auto const &axis : point->axes) { + checkNotFixed(axis); + } + } else if (auto const *unbinned = std::get_if(&data.body)) { + for (auto const &axis : unbinned->axes) { + if (auto const *continuous = std::get_if(&axis)) { + checkNotFixed(*continuous); + } + } + } auto const *binned = std::get_if(&data.body); if (!binned) { continue; } - const std::string path = at("data", i, data.name); std::optional nBinsOrNone = numBins(binned->axes, path); if (!nBinsOrNone) { continue; @@ -571,12 +699,6 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const } }; - auto checkConstraint = [&](std::string const &where, Constraint const &constraint) { - if (auto const *pdf = std::get_if(&constraint)) { - requireDistribution(pdf->pdf.name, where, "constraint"); - } - }; - // The constraint distributions of the gamma parameters of shape modifiers. auto checkGammaConstraint = [&](std::string const &where, GammaConstraint const &constraint) { auto const *pdfs = std::get_if(&constraint); @@ -629,12 +751,19 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const error(samplePath, "the name of the sample is not unique in the channel"); } checkSize(samplePath + ".data.contents", sample.contents.size(), "\"contents\""); - // These modifiers create objects that are named after them. + // These modifiers create objects that are named after them. Older versions of RooFit wrote a parameter that + // interpolates a sample more than once as several histosys modifiers of the same name, which are still read. std::set objectNames; - auto checkUniqueName = [&](std::string const &where, char const *type, std::string const &modifierName) { + auto checkUniqueName = [&](std::string const &where, char const *type, std::string const &modifierName, + bool allowedBeforeHS3v03 = false) { if (!objectNames.insert(std::string(type) + " " + modifierName).second) { - error(where, - std::string("there is another ") + type + " modifier named '" + modifierName + "' in the sample"); + const std::string message = + std::string("there is another ") + type + " modifier named '" + modifierName + "' in the sample"; + if (allowedBeforeHS3v03) { + strictError(where, message); + } else { + error(where, message); + } } }; if (sample.errors && sample.errors->size() != sample.contents.size()) { @@ -643,7 +772,9 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const ", contents=" + std::to_string(sample.contents.size())); } for (std::size_t iModifier = 0; iModifier < sample.modifiers.size(); ++iModifier) { - std::string modifierPath = samplePath + ".modifiers[" + std::to_string(iModifier) + "]"; + Modifier const &modifier = sample.modifiers[iModifier]; + const std::string modifierPath = + samplePath + ".modifiers[" + std::to_string(iModifier) + "](" + nameOf(modifier) + ")"; auto checkInterpolation = [&](std::optional const &interpolation, char const *type, std::string const &modifierName) { if (!interpolation && !dist.defaultInterpolation) { @@ -652,32 +783,34 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const "' is not given, and the channel has no \"default_interpolation\""); } }; + // The modifiers with a single parameter, which can have a constraint. Of the constraint types of older + // documents, RooFit only creates the Gaussian for them. + auto checkSingleParameter = [&](auto const &m) { + checkParameter(modifierPath, m.parameter.name, m.name); + if (auto const *pdf = std::get_if(&m.constraint)) { + requireDistribution(pdf->pdf.name, modifierPath, "constraint"); + } else if (auto const *type = std::get_if(&m.constraint); + type && *type != ConstraintType::Gauss) { + error(modifierPath, std::string("legacy constraint value '") + toString(*type) + "' for modifier '" + + m.name + + "' is a known constraint type, but it cannot be resolved in this context"); + } + checkCorrelated(modifierPath, m.parameter.name, m.constraint); + }; std::visit(overloaded{ - [&](NormFactor const &m) { - modifierPath += "(" + m.name + ")"; - checkParameter(modifierPath, m.parameter.name, m.name); - checkConstraint(modifierPath, m.constraint); - checkCorrelated(modifierPath, m.parameter.name, m.constraint); - }, + [&](NormFactor const &m) { checkSingleParameter(m); }, [&](NormSys const &m) { - modifierPath += "(" + m.name + ")"; - checkParameter(modifierPath, m.parameter.name, m.name); - checkConstraint(modifierPath, m.constraint); - checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkSingleParameter(m); checkInterpolation(m.interpolation, "normsys", m.name); }, [&](HistoSys const &m) { - modifierPath += "(" + m.name + ")"; - checkUniqueName(modifierPath, "histosys", m.name); - checkParameter(modifierPath, m.parameter.name, m.name); - checkConstraint(modifierPath, m.constraint); - checkCorrelated(modifierPath, m.parameter.name, m.constraint); + checkUniqueName(modifierPath, "histosys", m.name, true); + checkSingleParameter(m); checkInterpolation(m.interpolation, "histosys", m.name); checkSize(modifierPath + ".data.hi.contents", m.hi.contents.size(), "\"contents\""); checkSize(modifierPath + ".data.lo.contents", m.lo.contents.size(), "\"contents\""); }, [&](ShapeSys const &m) { - modifierPath += "(" + m.name + ")"; checkUniqueName(modifierPath, "shapesys", m.name); if (!m.vals.empty()) { checkSize(modifierPath + ".data.vals", m.vals.size(), "\"vals\""); @@ -691,22 +824,27 @@ void Validator::checkHistFactory(HistFactoryDist const &dist, std::string const checkGammaConstraint(modifierPath, m.constraint); }, [&](StatError const &m) { - modifierPath += "(" + m.name + ")"; if (!m.parameters.empty()) { checkSize(modifierPath + ".parameters", m.parameters.size(), "\"parameters\""); } checkGammaConstraint(modifierPath, m.constraint); checkStatErrorConstraints(modifierPath, m.constraint); + // A lognormal constraint makes no sense for gamma parameters. + if (auto const *type = std::get_if(&m.constraint); + type && *type == ConstraintType::LogNormal) { + error(modifierPath, std::string("unsupported constraint '") + toString(*type) + + "' of the staterror modifier in channel '" + name + + "', supported are \"Poisson\" and \"Gauss\""); + } }, [&](CustomModifier const &m) { - modifierPath += "(" + m.name + ")"; if (!isDefinedIn(m.name, "functions") && !isDefinedIn(m.name, "distributions") && !isExternal(m.name)) { error(modifierPath, "unable to find custom modifier '" + m.name + "'"); } }, }, - sample.modifiers[iModifier]); + modifier); } } } @@ -736,5 +874,14 @@ void validateHistFactoryDist(HistFactoryDist const &dist, std::string const &nam Validator{empty, diagnostics, options}.checkHistFactory(dist, name, ""); } +void validateCombinedDist(CombinedDist const &dist, std::string const &name, Diagnostics &diagnostics) +{ + // Without a document, the columns are categories of the workspace, whose labels are checked when it is built. + HS3Document empty; + ValidateOptions options; + options.isExternal = [](std::string const &) { return true; }; + Validator{empty, diagnostics, options}.checkCombined(dist, "", "the combined_dist '" + name + "'"); +} + } // namespace HS3 } // namespace RooFit diff --git a/roofit/hs3/src/HS3Variates.cxx b/roofit/hs3/src/HS3Variates.cxx new file mode 100644 index 0000000000000..2d455b03d7ee9 --- /dev/null +++ b/roofit/hs3/src/HS3Variates.cxx @@ -0,0 +1,1021 @@ +/* + * Project: RooFit + * Authors: + * Carsten D. Burgard, DESY/ATLAS, Nov 2026 + * Jonas Rembser, CERN, Nov 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +// HS3 requires the terms of a likelihood, and the components of a combined distribution or data set, to have distinct +// variates: a variate in two of them is one quantity modelled twice (e.g. a constraint that is a term of the likelihood +// and a factor of a channel), or two quantities with one name (e.g. the observable "mass" of two channels, which need +// their own names, "mass_SR" and "mass_CR"). All cases are checked with the same function, sharedVariates(), on the +// document, when it is exported and when it is imported. The generic distributions are read as far as the standard +// says what they consist of, through the nodes that the document refers to. +// +// This file also has what else follows from which distribution is paired with which data: how often a constraint +// contributes to a likelihood, and how a document is changed so that RooFit can read its auxiliary measurements and +// its point data. + +#include "HS3Document.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using RooFit::Detail::JSONNode; + +namespace RooFit { +namespace HS3 { +namespace { + +// Append a name to a list that has every name once. +void addUnique(std::vector &names, std::string const &name) +{ + if (std::find(names.begin(), names.end(), name) == names.end()) { + names.push_back(name); + } +} + +bool isName(JSONNode const &node) +{ + return !node.is_container() && !node.is_null() && !node.is_number(); +} + +// Add the names in `node` (one name or a list) to `out`. +void addNames(JSONNode const &node, std::set &out) +{ + if (node.is_seq()) { + for (JSONNode const &child : node.children()) { + addNames(child, out); + } + } else if (isName(node)) { + out.insert(node.val()); + } +} + +// The names in the list `key` of a generic distribution. The entries of the list are the names, or components that +// give the name as `field`. +std::vector namesIn(JSONNode const &node, const char *key, const char *field = nullptr) +{ + std::vector out; + if (JSONNode const *list = node.find(key)) { + for (JSONNode const &entry : list->children()) { + JSONNode const *name = field ? entry.find(field) : &entry; + if (name && isName(*name)) { + out.push_back(name->val()); + } + } + } + return out; +} + +// The constraint terms that the modifiers of a HistFactory distribution name in "constraints". The "constraint_name" +// of older documents is not counted: it predates the requirement of distinct variates. +std::vector namedConstraints(HistFactoryDist const &dist) +{ + std::vector out; + auto add = overloaded{[&](ConstraintPdf const &constraint) { + if (!constraint.legacyName) { + out.push_back(constraint.pdf.name); + } + }, + [&](GammaConstraintPdfs const &constraints) { + for (auto const &pdf : constraints.pdfs) { + if (pdf) { + out.push_back(pdf->name); + } + } + }, + [](auto const &) {}}; + for (Sample const &sample : dist.samples) { + for (Modifier const &modifier : sample.modifiers) { + std::visit(overloaded{[](CustomModifier const &) {}, [&](auto const &m) { std::visit(add, m.constraint); }}, + modifier); + } + } + return out; +} + +// The same for a HistFactory distribution that the document doesn't interpret, because a user registered their own +// importer or exporter for it. +std::vector namedConstraints(JSONNode const &dist) +{ + std::vector out; + if (JSONNode const *samples = dist.find("samples")) { + for (JSONNode const &sample : samples->children()) { + if (JSONNode const *modifiers = sample.find("modifiers")) { + for (JSONNode const &modifier : modifiers->children()) { + for (std::string const &constraint : namesIn(modifier, "constraints")) { + out.push_back(constraint); + } + } + } + } + } + return out; +} + +// The names of the components of a combined distribution. +std::vector componentsOf(CombinedDist const &dist) +{ + std::vector out; + for (auto const &component : dist.components) { + out.push_back(component.ref.name); + } + return out; +} + +// The node of a distribution that the document doesn't interpret, if it has one and, optionally, the type `type`. +JSONNode const *genericNode(Distribution const &dist, const char *type = nullptr) +{ + auto const *generic = std::get_if(&dist.body); + return generic && (!type || generic->type == type) ? generic->node : nullptr; +} + +// The distributions of a document by name, and what they say about the variates. The document must outlive this +// object. +class Distributions { +public: + explicit Distributions(HS3Document const &doc) : _doc{doc} + { + for (Distribution const &dist : doc.distributions) { + _byName.emplace(dist.name, &dist); + } + } + + // The distribution `name` if the document represents it as a `Body_t`, or nullptr. + template + Body_t const *find(std::string const &name) const + { + auto found = _byName.find(name); + return found != _byName.end() ? std::get_if(&found->second->body) : nullptr; + } + + // The node of the generic distribution `name` if it has the type `type`, or nullptr. + JSONNode const *nodeOf(std::string const &name, const char *type) const + { + auto found = _byName.find(name); + return found != _byName.end() ? genericNode(*found->second, type) : nullptr; + } + + // Add the variates of the distribution `name` to `out`: the "x" of a fundamental distribution (a name or a list) + // and the axes of a distribution with axes (e.g. histfactory_dist), through products, mixtures and combined + // distributions, and the "x" of the components of an array distribution. The constraint terms named by a + // histfactory_dist are terms of their own, see likelihoodTerms(). Distributions in `visited` are skipped. + void addVariates(std::string const &name, std::set &out, std::set &visited) const + { + if (!visited.insert(name).second) { + return; + } + if (auto const *histFactory = find(name)) { + for (BinnedAxis const &axis : histFactory->axes) { + out.insert(axis.name); + } + } + auto addVariatesOf = [&](std::vector const &parts) { + for (std::string const &part : parts) { + addVariates(part, out, visited); + } + }; + if (auto const *combined = find(name)) { + addVariatesOf(componentsOf(*combined)); + } + auto const *generic = find(name); + if (!generic || !generic->node) { + return; + } + JSONNode const &node = *generic->node; + if (generic->type == "product_dist") { + addVariatesOf(namesIn(node, "factors")); + } else if (generic->type == "mixture_dist") { + addVariatesOf(namesIn(node, "summands")); + } else if (generic->type == "array_dist") { + if (JSONNode const *components = node.find("components")) { + for (JSONNode const &component : components->children()) { + if (JSONNode const *x = component.find("x")) { + addNames(*x, out); + } + } + } + } else { + if (JSONNode const *x = node.find("x")) { + addNames(*x, out); + } + for (std::string const &axis : namesIn(node, "axes", "name")) { + out.insert(axis); + } + } + } + + // The terms of a likelihood, each once: its distributions (the components of a combined distribution, or of a + // RooSimultaneous written to ROOT_internal by older versions, are terms of their own), its "aux_distributions", + // and the constraint terms named by the modifiers of its histfactory_dist terms. + std::vector likelihoodTerms(Likelihood const &likelihood) const + { + std::vector terms; + auto add = [&](std::string const &name) { + auto legacy = _doc.rooFit.combinedDistributions.find(name); + if (auto const *combined = find(name)) { + for (std::string const &component : componentsOf(*combined)) { + addUnique(terms, component); + } + } else if (!_byName.count(name) && legacy != _doc.rooFit.combinedDistributions.end()) { + for (DistributionRef const &component : legacy->second.distributions) { + addUnique(terms, component.name); + } + } else { + addUnique(terms, name); + } + }; + for (LikelihoodTerm const &term : likelihood.terms) { + add(term.distribution.name); + } + for (DistributionRef const &aux : likelihood.auxDistributions) { + add(aux.name); + } + // The list grows by the constraint terms while it is read. + for (std::size_t i = 0; i < terms.size(); ++i) { + std::vector constraints; + if (auto const *histFactory = find(terms[i])) { + constraints = namedConstraints(*histFactory); + } else if (JSONNode const *generic = nodeOf(terms[i], "histfactory_dist")) { + constraints = namedConstraints(*generic); + } + for (std::string const &constraint : constraints) { + addUnique(terms, constraint); + } + } + return terms; + } + +private: + HS3Document const &_doc; + std::map _byName; +}; + +// The parts (terms or components) of something that must have distinct variates, with their variates. +using PartVariates = std::vector>>; + +// Append a message for each variate that several of `parts` share. `what` names the parts and `context` the whole, +// e.g. "terms" of "the likelihood 'lh'". +void addShared(PartVariates const &parts, std::string const &what, std::string const &context, + std::vector &messages) +{ + std::map> partsOfVariate; + std::vector order; + for (auto const &[part, variates] : parts) { + for (std::string const &variate : variates) { + auto &list = partsOfVariate[variate]; + if (list.empty()) { + order.push_back(variate); + } + addUnique(list, part); + } + } + for (std::string const &variate : order) { + auto const &list = partsOfVariate[variate]; + if (list.size() < 2) { + continue; + } + std::string names; + for (std::size_t i = 0; i < list.size(); ++i) { + names += (i == 0 ? "'" : (i + 1 == list.size() ? "' and '" : "', '")) + list[i]; + } + messages.push_back("the variate '" + variate + "' is shared by the " + what + " " + names + "' of " + context); + } +} + +// The axes of the components of a combined data set, by component. A component that is a combined data set itself has +// the axes of all of its components. +PartVariates componentAxes(HS3Document const &doc, CombinedData const &combined) +{ + PartVariates parts; + for (auto const &component : combined.components) { + std::set axes; + if (Data const *found = findByName(doc.data, component.ref.name)) { + std::visit(overloaded{[&](CombinedData const &inner) { + for (auto const &[name, innerAxes] : componentAxes(doc, inner)) { + axes.insert(innerAxes.begin(), innerAxes.end()); + } + }, + [&](PointData const &data) { + for (ContinuousAxis const &axis : data.axes) { + axes.insert(axis.name); + } + }, + [&](auto const &data) { + for (auto const &axis : data.axes) { + axes.insert(nameOf(axis)); + } + }}, + found->body); + } + parts.emplace_back(component.ref.name, axes); + } + return parts; +} + +// The number of entries of the data set `name` of the document (see Data::numEntries()). Without such a data set, the +// document is not valid, which is reported elsewhere: one is assumed. +int numDataEntries(HS3Document const &doc, std::string const &name) +{ + Data const *data = findByName(doc.data, name); + return data ? static_cast(data->numEntries()) : 1; +} + +} // namespace + +std::set variatesOf(HS3Document const &doc, Likelihood const &likelihood) +{ + Distributions distributions{doc}; + std::set variates; + std::set visited; + for (LikelihoodTerm const &term : likelihood.terms) { + distributions.addVariates(term.distribution.name, variates, visited); + if (auto const *combined = distributions.find(term.distribution.name)) { + variates.insert(combined->columns.begin(), combined->columns.end()); + } + } + return variates; +} + +std::set variatesOf(HS3Document const &doc, std::string const &distribution) +{ + std::set variates; + std::set visited; + Distributions{doc}.addVariates(distribution, variates, visited); + return variates; +} + +std::map> sharedAxes(HS3Document const &doc) +{ + std::map> out; + for (Data const &data : doc.data) { + if (auto const *combined = std::get_if(&data.body)) { + std::map counts; + for (auto const &[component, axes] : componentAxes(doc, *combined)) { + for (std::string const &axis : axes) { + if (++counts[axis] == 2) { + out[data.name].insert(axis); + } + } + } + } + } + return out; +} + +std::map> bookkeepingColumns(HS3Document const &doc) +{ + std::map> out; + if (doc.rooFit.bookkeepingColumns.empty()) { + return out; + } + Distributions distributions{doc}; + std::set variates; + std::set visited; + for (Distribution const &dist : doc.distributions) { + distributions.addVariates(dist.name, variates, visited); + } + for (auto const &[dataset, columns] : doc.rooFit.bookkeepingColumns) { + for (std::string const &column : columns) { + if (!variates.count(column)) { + out[dataset].insert(column); + } + } + } + return out; +} + +std::vector unlistedConstraints(HS3Document const &doc, std::vector *warnings) +{ + Distributions distributions{doc}; + std::vector messages; + for (Likelihood const &likelihood : doc.likelihoods) { + std::map contributions; + // The combined distributions in which a distribution is a component without data. + std::map> withoutData; + std::set named; + // The products that are being expanded, since a document that was not imported yet can have circular ones. + std::set expanding; + // `combined` is the combined distribution whose component without data the distribution is reached through. + std::function contribute = + [&](std::string const &name, int entries, std::string const &combined) { + contributions[name] += entries; + if (entries == 0) { + withoutData[name].insert(combined); + } + if (auto const *histFactory = distributions.find(name)) { + for (std::string const &constraint : namedConstraints(*histFactory)) { + named.insert(constraint); + } + } else if (JSONNode const *generic = distributions.nodeOf(name, "histfactory_dist")) { + for (std::string const &constraint : namedConstraints(*generic)) { + named.insert(constraint); + } + } else if (JSONNode const *product = distributions.nodeOf(name, "product_dist"); + product && expanding.insert(name).second) { + for (std::string const &factor : namesIn(*product, "factors")) { + contribute(factor, entries, combined); + } + expanding.erase(name); + } + }; + // The components of a combined distribution, each with the entries of the component of the paired combined data + // set that has the same key. Without paired data (a pure model), every component contributes once. + // A component that is a combined distribution itself (HS3 #161) contributes its own components, with the + // component of the data set that is paired with it. + std::function contributeCombined = + [&](std::string const &distName, CombinedDist const &dist, Data const *data) { + auto const *combinedData = data ? std::get_if(&data->body) : nullptr; + for (auto const &component : dist.components) { + int entries = 1; + Data const *componentData = nullptr; + if (combinedData) { + auto const *dataComponent = combinedData->find(component.key); + entries = dataComponent ? numDataEntries(doc, dataComponent->ref.name) : 0; + componentData = dataComponent ? findByName(doc.data, dataComponent->ref.name) : nullptr; + if (!dataComponent && warnings) { + warnings->push_back("the component '" + component.ref.name + "' of the combined distribution '" + + distName + "' has no component in the data set '" + data->name + + "' that the likelihood '" + likelihood.name + + "' pairs it with, so it does not contribute to the likelihood"); + } + } + if (auto const *nested = distributions.find(component.ref.name)) { + contributions[component.ref.name] += entries; + if (entries > 0) { + contributeCombined(component.ref.name, *nested, componentData); + } + continue; + } + contribute(component.ref.name, entries, distName); + } + }; + for (LikelihoodTerm const &term : likelihood.terms) { + // The paired data: a data set, a value given in-line, or nothing (a pure model). + auto const *dataRef = std::get_if(&term.data); + Data const *data = dataRef ? findByName(doc.data, dataRef->name) : nullptr; + std::string const &name = term.distribution.name; + if (auto const *combined = distributions.find(name)) { + ++contributions[name]; + contributeCombined(name, *combined, data); + } else { + contribute(name, data ? numDataEntries(doc, data->name) : 1, ""); + } + } + for (DistributionRef const &aux : likelihood.auxDistributions) { + contribute(aux.name, 1, ""); + } + for (std::string const &constraint : named) { + const int n = contributions[constraint]; + auto const ®ions = withoutData[constraint]; + std::string inRegions; + for (std::string const ®ion : regions) { + inRegions += (inRegions.empty() ? "'" : ", '") + region + "'"; + } + if (n != 1) { + messages.push_back("the constraint '" + constraint + + "' named by a modifier of a HistFactory channel contributes " + std::to_string(n) + + " times to the likelihood '" + likelihood.name + + "', but it must contribute exactly once" + + (n == 0 && !regions.empty() ? " (it is a component of the combined distribution " + + inRegions + " without data, which is no observation)" + : "")); + } else if (!regions.empty() && warnings) { + warnings->push_back("the constraint '" + constraint + "' is a term of the likelihood '" + likelihood.name + + "' and also a component of the combined distribution " + inRegions + + " without data. It contributes once, as the term"); + } + } + } + return messages; +} + +std::set componentsWithData(HS3Document const &doc) +{ + Distributions distributions{doc}; + std::set out; + // The components of the combined distribution `distName` that the combined data set `combinedData` has entries + // for, also those of a component that is a combined distribution itself (HS3 #161). + std::function collect = [&](std::string const &distName, + CombinedData const *combinedData) { + auto const *dist = distributions.find(distName); + if (!dist || !combinedData) { + return; + } + for (auto const &component : dist->components) { + auto const *dataComponent = combinedData->find(component.key); + if (dataComponent && numDataEntries(doc, dataComponent->ref.name) > 0) { + out.insert(component.ref.name); + collect(component.ref.name, find(doc.data, dataComponent->ref.name)); + } + } + }; + for (Likelihood const &likelihood : doc.likelihoods) { + for (LikelihoodTerm const &term : likelihood.terms) { + auto const *dataRef = std::get_if(&term.data); + collect(term.distribution.name, dataRef ? find(doc.data, dataRef->name) : nullptr); + } + } + return out; +} + +std::string arrayComponentForm(std::string const &componentType, JSONNode const &component, JSONNode const *defaults) +{ + // A value in canonical form: numbers as the double they represent (two literals are identical if they are the same + // number), names as they are, anything else as written. + auto canonical = [](JSONNode const &value) { + if (value.is_number()) { + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "%.17g", value.val_double()); + return std::string{"#"} + buffer; + } + if (value.has_val()) { + return "$" + value.val(); + } + std::stringstream ss; + value.writeJSON(ss); + return ss.str(); + }; + std::map fields; + if (defaults) { + for (auto const &field : defaults->children()) { + fields[field.key()] = canonical(field); + } + } + for (auto const &field : component.children()) { + fields[field.key()] = canonical(field); + } + std::string form = componentType; + for (auto const &[key, value] : fields) { + form += ";" + key + "=" + value; + } + return form; +} + +std::vector sharedVariates(HS3Document const &doc) +{ + Distributions distributions{doc}; + std::vector messages; + // The parts of a distribution, with their variates, as union composition (HS3) defines them: the components of a + // combined distribution, recursively, and the components of an array distribution are parts of their own; any other + // distribution is one part. Identical parts (the same distribution, or array components with the same canonical + // form, see arrayComponentForm()) are one part, named after the first of them ("[]" for an array + // component); the variates of such parts are added to `identical`. `reached` gets every distribution on the way. + // A combined distribution that is paired with the combined data set `data` has a factor only for the components with + // a component of the data set that has entries (HS3); without paired data, every component counts. + std::function &, + std::set &, std::set &)> + addUnionParts = [&](std::string const &name, Data const *data, PartVariates &parts, + std::map &nameOfForm, std::set &identical, + std::set &reached) { + reached.insert(name); + auto addPart = [&](std::string const &part, std::set const &variates) { + auto found = std::find_if(parts.begin(), parts.end(), [&](auto const &p) { return p.first == part; }); + if (found != parts.end()) { + identical.insert(variates.begin(), variates.end()); + return; + } + parts.emplace_back(part, variates); + }; + if (auto const *combined = distributions.find(name)) { + auto const *combinedData = data ? std::get_if(&data->body) : nullptr; + for (auto const &component : combined->components) { + Data const *componentData = nullptr; + if (combinedData) { + auto const *dataComponent = combinedData->find(component.key); + componentData = dataComponent ? findByName(doc.data, dataComponent->ref.name) : nullptr; + if (!componentData || numDataEntries(doc, componentData->name) == 0) { + continue; + } + } + addUnionParts(component.ref.name, componentData, parts, nameOfForm, identical, reached); + } + return; + } + JSONNode const *array = distributions.nodeOf(name, "array_dist"); + JSONNode const *components = array ? array->find("components") : nullptr; + if (!components) { + std::set variates; + std::set visited; + distributions.addVariates(name, variates, visited); + addPart(name, variates); + return; + } + std::string const componentType = array->has_child("component_type") ? (*array)["component_type"].val() : ""; + JSONNode const *defaults = array->find("default_parameters"); + for (JSONNode const &component : components->children()) { + std::set x; + if (JSONNode const *xNode = component.find("x")) { + addNames(*xNode, x); + } + std::string const form = arrayComponentForm(componentType, component, defaults); + auto found = nameOfForm.find(form); + if (found == nameOfForm.end()) { + found = nameOfForm.emplace(form, name + "[" + (x.empty() ? "" : *x.begin()) + "]").first; + } + addPart(found->second, x); + } + }; + // The variates of identical factors of a combined distribution, by name: they may occur in several sub-data sets of + // the paired combined data set. + std::map> identicalOfCombined; + auto unionOf = [&](std::string const &name, Data const *data, std::set &reached) { + PartVariates parts; + std::map nameOfForm; + std::set identical; + addUnionParts(name, data, parts, nameOfForm, identical, reached); + if (distributions.find(name)) { + identicalOfCombined[name] = identical; + } + return parts; + }; + std::set inLikelihoods; + for (Likelihood const &likelihood : doc.likelihoods) { + // The likelihood is the plain product of its terms (HS3): union composition applies within a term, not across + // terms, so that a variate of two terms is shared, also by identical factors. Each part is named after its term. + PartVariates parts; + std::set reached; + std::vector named; + auto addTerm = [&](std::string const &term, Data const *data) { + inLikelihoods.insert(term); + std::set reachedByTerm; + PartVariates termParts = unionOf(term, data, reachedByTerm); + // The same distribution as a part of another term is a factor of its own as well. + for (auto &part : termParts) { + bool again = std::any_of(parts.begin(), parts.end(), [&](auto const &p) { return p.first == part.first; }); + parts.emplace_back(again ? part.first + "' (again in the term '" + term : part.first, part.second); + } + reached.insert(reachedByTerm.begin(), reachedByTerm.end()); + }; + for (LikelihoodTerm const &term : likelihood.terms) { + auto const *dataRef = std::get_if(&term.data); + addTerm(term.distribution.name, dataRef ? findByName(doc.data, dataRef->name) : nullptr); + } + for (DistributionRef const &aux : likelihood.auxDistributions) { + addTerm(aux.name, nullptr); + } + // The constraint terms named by the HistFactory channels that no term reaches are terms of their own: RooFit adds + // them itself for documents before HS3 0.3. + for (std::string const &term : distributions.likelihoodTerms(likelihood)) { + if (!reached.count(term) && !inLikelihoods.count(term)) { + addTerm(term, nullptr); + } + } + addShared(parts, "terms", "the likelihood '" + likelihood.name + "'", messages); + } + for (Distribution const &dist : doc.distributions) { + if (std::holds_alternative(dist.body) && !inLikelihoods.count(dist.name)) { + std::set reached; + addShared(unionOf(dist.name, nullptr, reached), "components", "the combined_dist '" + dist.name + "'", + messages); + } + } + // The axes of the components of every combined data set, also of those that a likelihood lists. The axes for the + // variates of identical factors of the combined distribution that a likelihood pairs it with may occur in several + // components. + std::map> exempt = bookkeepingColumns(doc); + for (Likelihood const &likelihood : doc.likelihoods) { + for (LikelihoodTerm const &term : likelihood.terms) { + auto const *dataRef = std::get_if(&term.data); + auto identical = identicalOfCombined.find(term.distribution.name); + if (dataRef && identical != identicalOfCombined.end()) { + exempt[dataRef->name].insert(identical->second.begin(), identical->second.end()); + } + } + } + for (Data const &data : doc.data) { + auto const *combined = std::get_if(&data.body); + if (!combined) { + continue; + } + PartVariates parts = componentAxes(doc, *combined); + if (auto found = exempt.find(data.name); found != exempt.end()) { + for (auto &part : parts) { + for (std::string const &column : found->second) { + part.second.erase(column); + } + } + } + addShared(parts, "components", "the combined data set '" + data.name + "'", messages); + } + return messages; +} + +AuxiliaryConstraints readAuxiliaryChannelsAsConstraints(ValidatedHS3Document &validated, bool asChannels) +{ + HS3Document &doc = validated._doc; + AuxiliaryConstraints out; + // The components that are taken out of their combined distribution, by combined distribution, with their labels. + std::map> dropped; + std::set labels; + + // Components without data that are also terms with data. A data set can only be a component of one combined data + // set, so that the data of a term gets a copy for every further one. The copies are added to the document after + // the loop, which keeps pointers into the data sets. + std::vector copies; + auto hasEntries = [&](std::string const &name) { + Data const *data = findByName(doc.data, name); + data = data ? data : findByName(copies, name); + return data && data->numEntries() > 0; + }; + std::set dataOfComponents; + for (Likelihood &likelihood : doc.likelihoods) { + std::set termsToRemove; + for (LikelihoodTerm const &term : likelihood.terms) { + auto const *combinedDist = find(doc.distributions, term.distribution.name); + auto const *dataRef = std::get_if(&term.data); + auto *combined = dataRef ? find(doc.data, dataRef->name) : nullptr; + if (!combinedDist || !combined) { + continue; + } + for (auto const &component : combinedDist->components) { + if (component.key.size() != 1) { + continue; + } + std::string const &label = component.key.front(); + std::string const &dist = component.ref.name; + auto ownTerm = + std::find_if(likelihood.terms.begin(), likelihood.terms.end(), [&](LikelihoodTerm const &other) { + return other.distribution.name == dist && std::holds_alternative(other.data); + }); + if (ownTerm == likelihood.terms.end()) { + continue; + } + auto dataComponent = std::find_if(combined->components.begin(), combined->components.end(), + [&](auto const &c) { return c.key == component.key; }); + if (dataComponent != combined->components.end() && hasEntries(dataComponent->ref.name)) { + continue; // two contributions, which is not for RooFit to resolve + } + out.warnings.push_back("the distribution '" + dist + "' is a component of '" + term.distribution.name + + "' without data and a term of the likelihood '" + likelihood.name + + "' with data. It is read as " + + (asChannels ? "the component, with the data of the term." : "the term only.")); + if (asChannels) { + DataRef termData = std::get(ownTerm->data); + if (!dataOfComponents.insert(termData.name).second) { + Data const *original = findByName(doc.data, termData.name); + const std::string copyName = dataRef->name + "_" + label; + if (original && !findByName(doc.data, copyName) && !findByName(copies, copyName)) { + copies.push_back(*original); + copies.back().name = copyName; + termData.name = copyName; + } + } + if (dataComponent != combined->components.end()) { + dataComponent->ref = termData; + } else { + combined->components.push_back({{label}, termData}); + } + termsToRemove.insert(dist); + } else { + dropped[term.distribution.name][dist] = label; + labels.insert(label); + } + } + } + likelihood.terms.erase( + std::remove_if(likelihood.terms.begin(), likelihood.terms.end(), + [&](LikelihoodTerm const &term) { return termsToRemove.count(term.distribution.name) > 0; }), + likelihood.terms.end()); + } + doc.data.insert(doc.data.end(), copies.begin(), copies.end()); + if (asChannels) { + return out; + } + + for (auto const &item : doc.rooFit.auxiliaryChannels) { + auto const *combinedDist = find(doc.distributions, item.first); + for (auto const &ref : item.second) { + std::string label = ref.name; + if (combinedDist) { + for (auto const &component : combinedDist->components) { + if (component.ref.name == ref.name && component.key.size() == 1) { + label = component.key.front(); + } + } + } + dropped[item.first][ref.name] = label; + labels.insert(label); + } + } + auto isLabel = [&](std::vector const &key) { return key.size() == 1 && labels.count(key.front()); }; + for (Likelihood &likelihood : doc.likelihoods) { + std::vector added; + for (LikelihoodTerm const &term : likelihood.terms) { + auto found = dropped.find(term.distribution.name); + if (found == dropped.end()) { + continue; + } + auto const *dataRef = std::get_if(&term.data); + auto const *combined = dataRef ? find(doc.data, dataRef->name) : nullptr; + for (auto const &item : found->second) { + std::string const &dist = item.first; + std::string const &label = item.second; + if (std::any_of(likelihood.terms.begin(), likelihood.terms.end(), + [&](auto const &other) { return other.distribution.name == dist; })) { + continue; // a term of its own already + } + if (auto const *component = combined ? combined->find({label}) : nullptr) { + added.push_back({{dist}, component->ref}); + } else if (std::none_of(likelihood.auxDistributions.begin(), likelihood.auxDistributions.end(), + [&](auto const &ref) { return ref.name == dist; })) { + // Without data, the observed values are the values of the constant observables. + likelihood.auxDistributions.push_back({dist}); + } + } + } + likelihood.terms.insert(likelihood.terms.end(), added.begin(), added.end()); + } + for (Data &data : doc.data) { + if (auto *combined = std::get_if(&data.body)) { + auto &components = combined->components; + for (auto const &component : components) { + // Not the data sets without entries: they hold no values, and stay data sets of the workspace. + if (isLabel(component.key) && hasEntries(component.ref.name)) { + out.data.insert(component.ref.name); + } + } + components.erase(std::remove_if(components.begin(), components.end(), + [&](auto const &component) { return isLabel(component.key); }), + components.end()); + } + } + for (Domain &domain : doc.domains) { + for (auto &axis : domain.axes) { + if (auto *categorical = std::get_if(&axis)) { + auto &categories = categorical->categories; + categories.erase(std::remove_if(categories.begin(), categories.end(), + [&](std::string const &label) { return labels.count(label) > 0; }), + categories.end()); + } + } + } + for (auto const &droppedFrom : dropped) { + std::map const &components = droppedFrom.second; + for (auto const &item : components) { + out.terms.insert(item.first); + } + if (auto *combinedDist = find(doc.distributions, droppedFrom.first)) { + auto &all = combinedDist->components; + all.erase(std::remove_if(all.begin(), all.end(), + [&](auto const &component) { return components.count(component.ref.name) > 0; }), + all.end()); + } + } + return out; +} + +std::vector resolvePointData(ValidatedHS3Document &validated) +{ + HS3Document &doc = validated._doc; + std::vector errors; + auto soleVariateOf = [&](std::string const &distribution, std::string const &what) { + std::set const variates = variatesOf(doc, distribution); + if (variates.size() == 1) { + return *variates.begin(); + } + errors.push_back("the variate of the distribution '" + distribution + "', which " + what + + " is paired with, can't be determined, since it doesn't have a single variate"); + return std::string{}; + }; + std::vector inlineData; + std::map variateOfPointData; // of the point data sets without axes + for (Likelihood &likelihood : doc.likelihoods) { + for (LikelihoodTerm &term : likelihood.terms) { + std::string const &distribution = term.distribution.name; + if (auto const *value = std::get_if(&term.data)) { + // Older versions of RooFit wrote "data": [0], with the number 0 as the only entry, for a ModelConfig + // without data. In this exact form, it is no observation in documents before 0.3 (HS3 PR #154). RooFit + // now writes "data": null. + if (*value == 0. && likelihood.terms.size() == 1 && doc.metadata.version < Version{0, 3}) { + term.data = std::monostate{}; + continue; + } + std::string name = distribution + "_data"; + for (std::size_t k = 1; findByName(doc.data, name) || findByName(inlineData, name); ++k) { + name = distribution + "_data_" + std::to_string(k); + } + PointData point; + point.value = *value; + point.axes.push_back( + {soleVariateOf(distribution, "the number in the \"data\" of the likelihood '" + likelihood.name + "'"), + std::nullopt, std::nullopt}); + inlineData.push_back({name, std::move(point)}); + term.data = DataRef{name}; + continue; + } + auto const *ref = std::get_if(&term.data); + auto const *point = ref ? find(doc.data, ref->name) : nullptr; + if (!point) { + continue; + } + if (!point->axes.empty()) { + // The axis of point data is named like the variate of the paired distribution. + std::set const variates = variatesOf(doc, distribution); + if (variates.size() == 1 && point->axes.front().name != *variates.begin()) { + errors.push_back("the axis '" + point->axes.front().name + "' of the point data set '" + ref->name + + "' is not the variate '" + *variates.begin() + "' of the distribution '" + + distribution + "' it is paired with"); + } + continue; + } + std::string const variate = soleVariateOf(distribution, "the point data set '" + ref->name + "'"); + auto inserted = variateOfPointData.emplace(ref->name, variate); + if (!inserted.second && inserted.first->second != variate) { + errors.push_back("the point data set '" + ref->name + + "' without axes is paired with distributions of different variates, '" + + inserted.first->second + "' and '" + variate + "'"); + } + } + } + for (auto const &[name, variate] : variateOfPointData) { + find(doc.data, name)->axes.push_back({variate, std::nullopt, std::nullopt}); + } + doc.data.insert(doc.data.end(), std::make_move_iterator(inlineData.begin()), + std::make_move_iterator(inlineData.end())); + return errors; +} + +std::map> writeLikelihoodTermsOnce(HS3Document const &doc, JSONNode &root) +{ + std::map> removed; + if (!root.has_child("distributions")) { + return removed; + } + Distributions distributions{doc}; + // The products that are terms, with the factors that are terms wherever the product is one. + std::map> removable; + for (Likelihood const &likelihood : doc.likelihoods) { + std::vector const terms = distributions.likelihoodTerms(likelihood); + for (std::string const &term : terms) { + JSONNode const *product = distributions.nodeOf(term, "product_dist"); + if (!product) { + continue; + } + std::set factorsThatAreTerms; + for (std::string const &factor : namesIn(*product, "factors")) { + if (factor != term && std::find(terms.begin(), terms.end(), factor) != terms.end()) { + factorsThatAreTerms.insert(factor); + } + } + auto found = removable.find(term); + if (found == removable.end()) { + removable[term] = factorsThatAreTerms; + } else { + std::set both; + std::set_intersection(found->second.begin(), found->second.end(), factorsThatAreTerms.begin(), + factorsThatAreTerms.end(), std::inserter(both, both.begin())); + found->second = both; + } + } + } + // Products that another distribution uses are left alone. + for (Distribution const &dist : doc.distributions) { + if (JSONNode const *node = genericNode(dist)) { + for (const char *key : {"factors", "summands"}) { + for (std::string const &part : namesIn(*node, key)) { + removable.erase(part); + } + } + } + } + for (JSONNode &dist : root["distributions"].children()) { + auto found = removable.find(dist["name"].val()); + if (found == removable.end() || found->second.empty()) { + continue; + } + std::vector keep; + for (JSONNode const &factor : dist["factors"].children()) { + if (!found->second.count(factor.val())) { + keep.push_back(factor.val()); + } + } + if (!keep.empty()) { + dist["factors"].clear(); + dist["factors"].fill_seq(keep); + removed.insert(*found); + } + } + return removed; +} + +} // namespace HS3 +} // namespace RooFit diff --git a/roofit/hs3/src/HS3Writer.cxx b/roofit/hs3/src/HS3Writer.cxx index 9498fbde8fba8..ff3bdc16814f9 100644 --- a/roofit/hs3/src/HS3Writer.cxx +++ b/roofit/hs3/src/HS3Writer.cxx @@ -10,17 +10,14 @@ * listed in LICENSE (http://roofit.sourceforge.net/license.txt) */ -// Writes the typed representation of HS3 documents. The keys are the same as in HS3Parser.cxx, which the round-trip -// tests in testHS3Document.cxx check. +// Writes the typed representation of HS3 documents, with the keys that HS3Parser.cxx reads. #include "HS3Document.h" -#include "JSONIOUtils.h" #include #include #include -#include using RooFit::Detail::JSONNode; @@ -28,27 +25,49 @@ namespace RooFit { namespace HS3 { namespace { -// Infinite bounds are written as null. -void writeBound(JSONNode &node, double value) +// The counterparts of the readValue() functions of the parser, for the values that are written in one way only. +template +void writeValue(JSONNode &node, T const &value) { - if (std::isinf(value)) { - node.set_null(); - } else { - node << value; - } + node << value; } -void writeStrings(JSONNode &node, std::vector const &values) +template +void writeValue(JSONNode &node, Ref const &ref) { - node.fill_seq(values); + node << ref.name; } -template -void writeRefs(JSONNode &node, std::vector> const &refs) +void writeValue(JSONNode &node, NameOrNumber const &value) +{ + std::visit([&](auto const &v) { node << v; }, value); +} + +template +void writeValue(JSONNode &node, std::vector const &values) { node.set_seq(); - for (auto const &ref : refs) { - node.append_child() << ref.name; + for (auto const &value : values) { + writeValue(node.append_child(), value); + } +} + +// A component that is left out if it has no value. +template +void writeOptional(JSONNode &node, const char *key, std::optional const &value) +{ + if (value) { + writeValue(node[key], *value); + } +} + +// Writes a number, as an integer if it has no fractional part. +void writeNumber(JSONNode &node, double value) +{ + if (static_cast(value) == value) { + node << static_cast(value); + } else { + node << value; } } @@ -68,27 +87,17 @@ void writeBinning(JSONNode &node, Binning const &binning) } } -void writeAxis(JSONNode &node, BinnedAxis const &axis) -{ - node.set_map(); - node["name"] << axis.name; - // The range of an irregular binning is given by its "edges": HS3 0.3 forbids "min" and "max" alongside them. - if (!std::holds_alternative(axis.binning)) { - node["min"] << axis.min; - node["max"] << axis.max; - } - writeBinning(node, axis.binning); -} - +// An infinite or missing bound of a continuous data axis is omitted, which means that the range is unbounded. JSON null +// is not defined for bounds. void writeAxis(JSONNode &node, ContinuousAxis const &axis) { node.set_map(); node["name"] << axis.name; - if (axis.min) { - writeBound(node["min"], *axis.min); + if (axis.min && !std::isinf(*axis.min)) { + node["min"] << *axis.min; } - if (axis.max) { - writeBound(node["max"], *axis.max); + if (axis.max && !std::isinf(*axis.max)) { + node["max"] << *axis.max; } } @@ -96,7 +105,7 @@ void writeAxis(JSONNode &node, CategoricalAxis const &axis) { node.set_map(); node["name"] << axis.name; - writeStrings(node["categories"], axis.categories); + node["categories"].fill_seq(axis.categories); } // An infinite or missing bound of a domain axis is omitted, which means that the range is unbounded. @@ -105,7 +114,7 @@ void writeAxis(JSONNode &node, DomainAxis const &axis) node.set_map(); node["name"] << axis.name; if (axis.fixedValue) { - node[fixedValueKey] << *axis.fixedValue; + node["value"] << *axis.fixedValue; return; } // HS3 0.3 forbids "min" and "max" alongside "edges", which give the range. They are only written if they differ from @@ -170,23 +179,14 @@ void writeGammaConstraint(JSONNode &node, GammaConstraint const &constraint, con constraint); } -JSONNode &appendModifier(JSONNode &modifiers, std::string const &name, const char *type) -{ - JSONNode &mod = modifiers.append_child().set_map(); - mod["name"] << name; - mod["type"] << type; - return mod; -} - -void writeModifier(JSONNode &modifiers, Modifier const &modifier) +void writeModifier(JSONNode &mod, Modifier const &modifier) { + mod.set_map(); + mod["name"] << nameOf(modifier); + mod["type"] << typeOf(modifier); std::visit(overloaded{ - [&](NormFactor const &m) { - JSONNode &mod = appendModifier(modifiers, m.name, "normfactor"); - writeParameter(mod, m.parameter, m.constraint); - }, + [&](NormFactor const &m) { writeParameter(mod, m.parameter, m.constraint); }, [&](NormSys const &m) { - JSONNode &mod = appendModifier(modifiers, m.name, "normsys"); writeParameter(mod, m.parameter, m.constraint); if (m.interpolation) { writeInterpolation(mod["interpolation"], *m.interpolation); @@ -196,7 +196,6 @@ void writeModifier(JSONNode &modifiers, Modifier const &modifier) data["hi"] << m.hi; }, [&](HistoSys const &m) { - JSONNode &mod = appendModifier(modifiers, m.name, "histosys"); writeParameter(mod, m.parameter, m.constraint); if (m.interpolation) { writeInterpolation(mod["interpolation"], *m.interpolation); @@ -206,8 +205,7 @@ void writeModifier(JSONNode &modifiers, Modifier const &modifier) writeContents(data["hi"].set_map()["contents"], m.hi.contents); }, [&](ShapeSys const &m) { - JSONNode &mod = appendModifier(modifiers, m.name, "shapefactor"); - writeRefs(mod["parameters"], m.parameters); + writeValue(mod["parameters"], m.parameters); writeGammaConstraint(mod, m.constraint, "constraint"); // The values are only used by the constraint types of older documents. if (!m.vals.empty()) { @@ -215,15 +213,27 @@ void writeModifier(JSONNode &modifiers, Modifier const &modifier) } }, [&](StatError const &m) { - JSONNode &mod = appendModifier(modifiers, m.name, "staterror"); - writeRefs(mod["parameters"], m.parameters); + writeValue(mod["parameters"], m.parameters); writeGammaConstraint(mod, m.constraint, "constraint"); }, - [&](CustomModifier const &m) { appendModifier(modifiers, m.name, "custom"); }, + [](CustomModifier const &) {}, }, modifier); } +// The columns and the components of a combined data set or distribution, which name what they select in `refKey`. +template +void writeCombined(JSONNode &node, Combined const &combined, const char *refKey) +{ + node["columns"].fill_seq(combined.columns); + JSONNode &components = node["components"].set_seq(); + for (auto const &component : combined.components) { + JSONNode &out = components.append_child().set_map(); + out["key"].fill_seq(component.key); + out[refKey] << component.ref.name; + } +} + void writeParameterPoint(JSONNode &points, ParameterPoint const &point) { JSONNode ¶meters = appendNamed(points, point.name)["parameters"].set_seq(); @@ -244,13 +254,7 @@ void writeData(JSONNode &dataArray, Data const &data) std::visit(overloaded{ [&](BinnedData const &d) { node["type"] << "binned"; - writeAxes(node["axes"], d.axes); - writeContents(node["contents"], d.contents); - if (d.uncertainty) { - JSONNode &uncertainty = node["uncertainty"].set_map(); - uncertainty["type"] << "gaussian_uncertainty"; - uncertainty["sigma"].fill_seq(d.uncertainty->sigma); - } + writeBinnedData(node, d); }, [&](UnbinnedData const &d) { node["type"] << "unbinned"; @@ -268,26 +272,16 @@ void writeData(JSONNode &dataArray, Data const &data) } } } - if (d.weights) { - node["weights"].fill_seq(*d.weights); - } + writeOptional(node, "weights", d.weights); }, [&](CombinedData const &d) { node["type"] << "combined"; - writeStrings(node["columns"], d.columns); - JSONNode &components = node["components"].set_seq(); - for (CombinedDataComponent const &component : d.components) { - JSONNode &out = components.append_child().set_map(); - writeStrings(out["key"], component.key); - out["dataset"] << component.data.name; - } + writeCombined(node, d, "dataset"); }, [&](PointData const &d) { node["type"] << "point"; node["value"] << d.value; - if (d.uncertainty) { - node["uncertainty"] << *d.uncertainty; - } + writeOptional(node, "uncertainty", d.uncertainty); if (!d.axes.empty()) { writeAxes(node["axes"], d.axes); } @@ -300,101 +294,80 @@ void writeLikelihood(JSONNode &likelihoods, Likelihood const &likelihood) { JSONNode &node = appendNamed(likelihoods, likelihood.name); JSONNode &distributions = node["distributions"].set_seq(); - JSONNode &data = node["data"].set_seq(); - for (auto const &term : likelihood.terms) { - distributions.append_child() << term.distribution.name; - std::visit(overloaded{[&](DataRef const &ref) { data.append_child() << ref.name; }, - [&](double value) { writeNumber(data.append_child(), value); }}, - term.data); + if (likelihood.pureModel) { + for (auto const &term : likelihood.terms) { + distributions.append_child() << term.distribution.name; + } + node["data"].set_null(); + } else { + JSONNode &data = node["data"].set_seq(); + for (auto const &term : likelihood.terms) { + distributions.append_child() << term.distribution.name; + std::visit(overloaded{[&](DataRef const &ref) { data.append_child() << ref.name; }, + [&](double value) { writeNumber(data.append_child(), value); }, + [&](std::monostate) { data.append_child().set_null(); }}, + term.data); + } } if (!likelihood.auxDistributions.empty()) { - writeRefs(node["aux_distributions"], likelihood.auxDistributions); + writeValue(node["aux_distributions"], likelihood.auxDistributions); } } void writeAnalysis(JSONNode &analyses, Analysis const &analysis) { JSONNode &node = appendNamed(analyses, analysis.name); - writeRefs(node["domains"], analysis.domains); + writeValue(node["domains"], analysis.domains); node["likelihood"] << analysis.likelihood.name; if (!analysis.parametersOfInterest.empty()) { - writeRefs(node["parameters_of_interest"], analysis.parametersOfInterest); - } - if (analysis.init) { - node["init"] << analysis.init->name; - } - if (analysis.prior) { - node["prior"] << analysis.prior->name; + writeValue(node["parameters_of_interest"], analysis.parametersOfInterest); } + writeOptional(node, "init", analysis.init); + writeOptional(node, "prior", analysis.prior); } -void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) +void writeValue(JSONNode &node, Attributes const &attributes) { - auto rootInternal = [&]() -> JSONNode & { return root.get("misc", "ROOT_internal"); }; - if (!internal.attributes.empty()) { - JSONNode &attributes = rootInternal()["attributes"].set_map(); - for (auto const &item : internal.attributes) { - JSONNode &node = attributes[item.first].set_map(); - if (!item.second.dict.empty()) { - JSONNode &dict = node["dict"].set_map(); - for (auto const &entry : item.second.dict) { - dict[entry.first] << entry.second; - } - } - if (!item.second.tags.empty()) { - writeStrings(node["tags"], item.second.tags); - } - } - } - if (!internal.combinedDistributions.empty()) { - JSONNode &combined = rootInternal()["combined_distributions"].set_map(); - for (auto const &item : internal.combinedDistributions) { - JSONNode &node = combined[item.first].set_map(); - node["index_cat"] << item.second.indexCat; - writeStrings(node["labels"], item.second.labels); - node["indices"].fill_seq(item.second.indices); - writeRefs(node["distributions"], item.second.distributions); + node.set_map(); + if (!attributes.dict.empty()) { + JSONNode &dict = node["dict"].set_map(); + for (auto const &entry : attributes.dict) { + dict[entry.first] << entry.second; } } - if (!internal.combinedDatasets.empty()) { - JSONNode &combined = rootInternal()["combined_datasets"].set_map(); - for (auto const &item : internal.combinedDatasets) { - JSONNode &node = combined[item.first].set_map(); - node["index_cat"] << item.second.indexCat; - writeStrings(node["labels"], item.second.labels); - node["indices"].fill_seq(item.second.indices); - } + if (!attributes.tags.empty()) { + node["tags"].fill_seq(attributes.tags); } - if (!internal.modelConfigs.empty()) { - JSONNode &modelConfigs = rootInternal()["ModelConfigs"].set_map(); - for (auto const &item : internal.modelConfigs) { - JSONNode &node = modelConfigs[item.first].set_map(); - if (item.second.mcName) { - node["mcName"] << *item.second.mcName; - } - if (item.second.pdfName) { - node["pdfName"] << *item.second.pdfName; - } - if (item.second.combinedDataName) { - node["combined_data_name"] << *item.second.combinedDataName; - } - if (item.second.observables) { - node["observables"].fill_seq(*item.second.observables); - } - } - } - if (!internal.channelConstraints.empty()) { - JSONNode &constraints = rootInternal()["histfactory_constraints"].set_map(); - for (auto const &item : internal.channelConstraints) { - writeRefs(constraints[item.first], item.second); - } +} + +void writeValue(JSONNode &node, ModelConfigInfo const &info) +{ + node.set_map(); + writeOptional(node, "mcName", info.mcName); + writeOptional(node, "pdfName", info.pdfName); + writeOptional(node, "observables", info.observables); +} + +// A component of "misc.ROOT_internal" with one value per name, which is left out if there are no values. +template +void writeInternal(JSONNode &root, const char *key, std::map const &values) +{ + if (values.empty()) { + return; } - if (!internal.bookkeepingColumns.empty()) { - JSONNode &columns = rootInternal()["bookkeeping_columns"].set_map(); - for (auto const &item : internal.bookkeepingColumns) { - writeStrings(columns[item.first], item.second); - } + JSONNode &node = root.get("misc", "ROOT_internal")[key].set_map(); + for (auto const &item : values) { + writeValue(node[item.first], item.second); } +} + +void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) +{ + writeInternal(root, "attributes", internal.attributes); + writeInternal(root, "ModelConfigs", internal.modelConfigs); + writeInternal(root, "histfactory_constraints", internal.channelConstraints); + writeInternal(root, "auxiliary_channels", internal.auxiliaryChannels); + writeInternal(root, "bookkeeping_columns", internal.bookkeepingColumns); if (!internal.stepWidths.empty()) { JSONNode &stepWidths = root.get("misc", "minimization")["parameter_stepwidths"].set_seq(); for (auto const &stepWidth : internal.stepWidths) { @@ -405,16 +378,6 @@ void writeRooFitInternal(JSONNode &root, RooFitInternal const &internal) } // namespace -void writeNumber(JSONNode &node, double value) -{ - // To make sure there are no unnecessary floating points in the JSON - if (static_cast(value) == value) { - node << static_cast(value); - } else { - node << value; - } -} - void writeContents(JSONNode &node, std::vector const &values) { node.set_seq(); @@ -437,26 +400,6 @@ void writeInterpolation(JSONNode &node, Interpolation const &interpolation) namespace { -void writeValue(JSONNode &node, NameOrNumber const &value) -{ - std::visit([&](auto const &v) { node << v; }, value); -} - -template -void writeValue(JSONNode &node, T const &value) -{ - node << value; -} - -template -void writeList(JSONNode &node, std::vector const &values) -{ - node.set_seq(); - for (auto const &value : values) { - writeValue(node.append_child(), value); - } -} - template void writeInterpolated(JSONNode &node, Function_t const &function, char const *type) { @@ -469,14 +412,37 @@ void writeInterpolated(JSONNode &node, Function_t const &function, char const *t ++i) { writeInterpolation(interpolationsNode.append_child(), interpolations[i]); } - writeList(node["vars"], function.parameters); + writeValue(node["vars"], function.parameters); node["nom"] << function.nominal; - writeList(node["high"], function.high); - writeList(node["low"], function.low); + writeValue(node["high"], function.high); + writeValue(node["low"], function.low); } } // namespace +void writeAxis(JSONNode &node, BinnedAxis const &axis) +{ + node.set_map(); + node["name"] << axis.name; + // The range of an irregular binning is given by its "edges": HS3 0.3 forbids "min" and "max" alongside them. + if (!std::holds_alternative(axis.binning)) { + node["min"] << axis.min; + node["max"] << axis.max; + } + writeBinning(node, axis.binning); +} + +void writeBinnedData(JSONNode &node, BinnedData const &data) +{ + writeAxes(node["axes"], data.axes); + writeContents(node["contents"], data.contents); + if (data.uncertainty) { + JSONNode &uncertainty = node["uncertainty"].set_map(); + uncertainty["type"] << "gaussian_uncertainty"; + uncertainty["sigma"].fill_seq(data.uncertainty->sigma); + } +} + void writeInterpolatedFunction(JSONNode &node, InterpolatedFunction const &function) { writeInterpolated(node, function, "interpolation"); @@ -504,7 +470,7 @@ void writeHistFactoryDist(JSONNode &node, HistFactoryDist const &dist) JSONNode &s = appendNamed(samples, sample.name); JSONNode &modifiers = s["modifiers"].set_seq(); for (auto const &modifier : sample.modifiers) { - writeModifier(modifiers, modifier); + writeModifier(modifiers.append_child(), modifier); } JSONNode &data = s["data"].set_map(); writeContents(data["contents"], sample.contents); @@ -514,6 +480,12 @@ void writeHistFactoryDist(JSONNode &node, HistFactoryDist const &dist) } } +void writeCombinedDist(JSONNode &node, CombinedDist const &dist) +{ + node["type"] << "combined_dist"; + writeCombined(node, dist, "dist"); +} + void writeDocument(JSONNode &root, HS3Document const &doc) { root.set_map(); diff --git a/roofit/hs3/src/JSONFactories_HistFactory.cxx b/roofit/hs3/src/JSONFactories_HistFactory.cxx index 5394884fcbc46..5a7a711bb6e18 100644 --- a/roofit/hs3/src/JSONFactories_HistFactory.cxx +++ b/roofit/hs3/src/JSONFactories_HistFactory.cxx @@ -36,7 +36,6 @@ #include #include -#include #include #include #include @@ -47,16 +46,13 @@ #include #include -#include "static_execute.h" #include "HS3Builder.h" #include "JSONIOUtils.h" using RooFit::Detail::JSONNode; -using RooFit::HS3::additivePolynomialLinear; using RooFit::HS3::Interpolation; using RooFit::HS3::InterpolationClass; using RooFit::HS3::multiplicativePiecewiseExponential; -using RooFit::HS3::multiplicativePolynomialExponential; namespace HS3 = RooFit::HS3; @@ -77,16 +73,13 @@ double round_prec(double d, int nSig) return sf * std::round(d / sf); } +// An interpolation as it is written in the document, for messages. std::string interpolationString(const Interpolation &interpolation) { + auto tree = RooFit::Detail::JSONTree::create(); + HS3::writeInterpolation(tree->rootnode(), interpolation); std::stringstream ss; - ss << R"({"type":")" << toString(interpolation.type) << R"(","in":")" << toString(interpolation.in) << R"(","out":)"; - if (interpolation.out) { - ss << '"' << toString(*interpolation.out) << '"'; - } else { - ss << "null"; - } - ss << '}'; + tree->rootnode().writeJSON(ss); return ss.str(); } @@ -139,29 +132,6 @@ std::vector toCodes(std::vector const &interpolations, Inter return out; } -// Reports the problems with an object that doesn't come with a document. -void report(std::string const &name, HS3::Diagnostics &diagnostics) -{ - for (auto &diagnostic : diagnostics) { - diagnostic.path = name + (diagnostic.path.empty() ? "" : "." + diagnostic.path); - } - HS3::report(diagnostics); -} - -int interpolationCode(std::optional const &interpolation, - std::optional const &defaultInterpolation, InterpolationClass interpolationClass, - std::string const &context) -{ - if (interpolation) { - return codeFromInterpolation(*interpolation, interpolationClass, context); - } - // HS3 forbids implementation-specific default interpolations. - if (!defaultInterpolation) { - RooJSONFactoryWSTool::error(context + " is not given, and the channel has no \"default_interpolation\""); - } - return codeFromInterpolation(*defaultInterpolation, interpolationClass, context); -} - void erasePrefix(std::string &str, std::string_view prefix) { if (startsWith(str, prefix)) { @@ -185,16 +155,22 @@ void sortByName(Coll &coll) std::sort(coll.begin(), coll.end(), [](auto &l, auto &r) { return l.name < r.name; }); } +// Whether any of the gamma parameters names a constraint. Otherwise, they are all free, and nothing is written. +bool anyNamed(HS3::GammaConstraintPdfs const &constraints) +{ + return std::any_of(constraints.pdfs.begin(), constraints.pdfs.end(), + [](auto const &pdf) { return pdf.has_value(); }); +} + template T *findClient(RooAbsArg *gamma) { for (const auto &client : gamma->clients()) { if (auto casted = dynamic_cast(client)) { return casted; - } else { - T *c = findClient(client); - if (c) - return c; + } + if (T *found = findClient(client)) { + return found; } } return nullptr; @@ -231,14 +207,9 @@ RooAbsPdf *findConstraint(RooAbsArg *g, std::vector const &channelC return findClient(g); } -inline std::string defaultGammaName(std::string const &sysname, std::size_t i) -{ - return "gamma_" + sysname + "_bin_" + std::to_string(i); -} - -RooRealVar &createNominal(RooWorkspace &ws, std::string const &parname, double val, double min, double max) +RooRealVar &createNominal(RooWorkspace &ws, std::string const &parname, double val) { - RooRealVar &nom = getOrCreate(ws, "nom_" + parname, val, min, max); + RooRealVar &nom = getOrCreate(ws, "nom_" + parname, val, -10, 10); nom.setConstant(true); return nom; } @@ -274,20 +245,16 @@ RooAbsPdf &arrayComponentFor(RooProdPdf const &array, RooAbsArg const ¶m, st // Find the constraint distribution `constraintName` of the parameter `param`. If the name resolves to an array // distribution, the component that depends on the parameter is returned: that is the term that enters the likelihood -// for this parameter, and the whole array enters once through the components of all of its parameters. Returns -// nullptr if the name cannot be resolved. -RooAbsPdf *findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &constraintName, RooAbsArg const ¶m, +// for this parameter, and the whole array enters once through the components of all of its parameters. +RooAbsPdf &findNamedConstraint(RooJSONFactoryWSTool &tool, std::string const &constraintName, RooAbsArg const ¶m, std::string const &context) { RooAbsPdf *constraint = tool.tryRequest(constraintName); - if (!constraint || !constraint->getAttribute(arrayDistTag)) { - return constraint; + if (!constraint) { + RooJSONFactoryWSTool::error("unable to find the constraint '" + constraintName + "' of " + context); } auto const *array = dynamic_cast(constraint); - if (!array) { - return constraint; - } - return &arrayComponentFor(*array, param, context); + return array && array->getAttribute(arrayDistTag) ? arrayComponentFor(*array, param, context) : *constraint; } // The parameter of a modifier, which is created if the document doesn't define it. It's a variable, or a constant if @@ -323,31 +290,22 @@ void addConstraint(RooJSONFactoryWSTool &tool, HS3::Constraint const &constraint RooWorkspace &ws = *tool.workspace(); std::visit(HS3::overloaded{ [](HS3::Unconstrained) {}, - [&](HS3::ConstraintType type) { - if (type != HS3::ConstraintType::Gauss) { - RooJSONFactoryWSTool::error(std::string("legacy constraint value '") + toString(type) + - "' for modifier '" + modifier + - "' is a known constraint type, but it cannot be resolved in this " - "context"); - } - RooRealVar &nom = createNominal(ws, param.GetName(), nominal, -10, 10); + [&](HS3::ConstraintType) { + // The validation only lets a Gaussian constraint through. + RooRealVar &nom = createNominal(ws, param.GetName(), nominal); setErrorIfVariable(param, 1.0); const std::string name = constraintName(param.GetName()); RooAbsPdf *pdf = ws.pdf(name); constraints.add(pdf ? *pdf : tool.wsEmplace(name, param, nom, 1.)); }, [&](HS3::ConstraintPdf const &constraintPdf) { - RooAbsPdf *pdf = findNamedConstraint(tool, constraintPdf.pdf.name, param, + RooAbsPdf &pdf = findNamedConstraint(tool, constraintPdf.pdf.name, param, std::string("the ") + modifierType + " modifier '" + modifier + "' of sample '" + sample + "'"); - if (!pdf) { - RooJSONFactoryWSTool::error("unable to find definition of of constraint '" + - constraintPdf.pdf.name + "' for modifier '" + modifier + "'"); - } - if (auto gauss = dynamic_cast(pdf)) { + if (auto gauss = dynamic_cast(&pdf)) { setErrorIfVariable(param, gauss->getSigma().getVal()); } - constraints.add(*pdf); + constraints.add(pdf); }, }, constraint); @@ -362,7 +320,7 @@ void addSysConstraint(RooJSONFactoryWSTool &tool, HS3::Constraint const &constra { auto const *pdf = std::get_if(&constraint); if (pdf && pdf->legacyName) { - createNominal(*tool.workspace(), param.GetName(), 0.0, -10, 10); + createNominal(*tool.workspace(), param.GetName(), 0.0); } addConstraint(tool, constraint, param, 0.0, modifierType, modifier, sample, constraints); } @@ -376,24 +334,13 @@ std::optional findPoissonTau(RooPoisson const &constraint, RooAbsArg con return std::nullopt; } + // The tau is the constant factor. In imported workspaces, it can also be a variable that is named like it. for (RooAbsArg *arg : mean->servers()) { - if (arg == &gamma) { - continue; - } - - if (auto const *tau = dynamic_cast(arg)) { + auto const *tau = dynamic_cast(arg); + if (tau && arg != &gamma && (tau->isConstant() || endsWith(tau->GetName(), "_tau"))) { return tau->getVal(); } - - // Imported workspaces can sometimes represent - // constants as constant RooRealVars. - if (auto const *real = dynamic_cast(arg)) { - if (real->isConstant() || endsWith(std::string(real->GetName()), "_tau")) { - return real->getVal(); - } - } } - return std::nullopt; } @@ -451,34 +398,29 @@ void configureGammasWithConstraints(RooArgList const &gammas, std::vector const &parnames, std::vector const &vals, HS3::GammaConstraint const &constraint, RooArgSet &constraints, RooArgSet const &observables, - double gammaMax, double minSigma, std::size_t nBins, std::string const &context) + double gammaMax, double minSigma, std::size_t nBins, std::string const &context, + bool configureGammas = true) { RooWorkspace &ws = *tool.workspace(); // The validation made sure that there is a value, parameter and constraint for each bin, if they are given. RooArgList gammas; for (std::size_t i = 0; i < nBins; ++i) { - const std::string name = parnames.empty() ? defaultGammaName(sysname, i) : parnames[i].name; - if (auto *existing = dynamic_cast(ws.obj(name))) { - gammas.add(*existing); - } else { - gammas.add(getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); - } + const std::string name = parnames.empty() ? "gamma_" + sysname + "_bin_" + std::to_string(i) : parnames[i].name; + auto *existing = dynamic_cast(ws.obj(name)); + gammas.add(existing ? *existing : getOrCreate(ws, name, 1., defaultGammaMin, gammaMax)); } // The gammas and `vals` are given in the order of HS3 (last axis fastest), which is also the order in which they // are paired below. ParamHistFunc expects its parameters in TH1 order. - checkParamHistFuncOrderIsDefined(tool.hs3VersionInput(), observables, "the modifier '" + sysname + "'"); + checkParamHistFuncOrderIsDefined(tool.hs3Version(), observables, "the modifier '" + sysname + "'"); RooArgList gammasInTH1Order = toParamHistFuncOrder(gammas, observables); // Shape modifiers of the same name in a channel share the function, so they need the same parameters. if (RooAbsArg *existing = ws.arg(phfname.c_str())) { auto *existingPHF = dynamic_cast(existing); - bool same = existingPHF && existingPHF->paramList().size() == gammasInTH1Order.size(); - for (std::size_t i = 0; same && i < gammasInTH1Order.size(); ++i) { - same = &existingPHF->paramList()[i] == &gammasInTH1Order[i]; - } - if (!same) { + if (!existingPHF || existingPHF->paramList().size() != gammasInTH1Order.size() || + !existingPHF->paramList().hasSameLayout(gammasInTH1Order)) { RooJSONFactoryWSTool::error("the modifier '" + sysname + "' has other parameters than another modifier " + "of this name in the channel"); } @@ -487,8 +429,7 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, auto addGammaConstraints = [&](Constraint::Type type) { for (auto const &term : createGammaConstraints(gammas, vals, minSigma, type).constraints) { - ws.import(*term, RooFit::RecycleConflictNodes(), RooFit::Silence()); - constraints.add(*ws.pdf(term->GetName())); + constraints.add(tool.wsImport(*term)); } }; @@ -498,18 +439,15 @@ ParamHistFunc &createPHF(RooJSONFactoryWSTool &tool, std::string const &phfname, std::vector constraintPdfs; for (std::size_t i = 0; i < pdfs->pdfs.size(); ++i) { auto const &pdfName = pdfs->pdfs[i]; - RooAbsPdf *pdf = nullptr; - if (pdfName) { - pdf = findNamedConstraint(tool, pdfName->name, gammas[i], context); - if (!pdf) { - RooJSONFactoryWSTool::error("unable to find definition of constraint '" + pdfName->name + - "' for modifier '" + sysname + "'"); - } + RooAbsPdf *pdf = pdfName ? &findNamedConstraint(tool, pdfName->name, gammas[i], context) : nullptr; + if (pdf) { constraints.add(*pdf); } constraintPdfs.push_back(pdf); } - configureGammasWithConstraints(gammas, vals, constraintPdfs, minSigma); + // Without `configureGammas`, nothing is derived from `vals`: the gammas keep their range and constness, and + // only get the initial error that their constraint distribution encodes. + configureGammasWithConstraints(gammas, configureGammas ? vals : std::vector{}, constraintPdfs, minSigma); } if (vals.empty()) { @@ -555,7 +493,6 @@ double constraintRelError(RooAbsPdf const &constraint, RooAbsArg const &gamma) gamma.GetName() + "': only RooGaussian and RooPoisson with mean gamma * tau are supported for the " "relative uncertainties of gammas"); - return std::numeric_limits::quiet_NaN(); } struct ChannelContext { @@ -564,28 +501,11 @@ struct ChannelContext { RooArgSet const &observables; std::string fprefix; std::string channelName; - std::size_t nBins; RooAbsArg const *mcStatObject; RooArgSet &constraints; }; -// Collects what the modifiers of a sample contribute to the RooFit objects. -struct SampleElements { - RooArgList shapeElems; - RooArgList normElems; - - RooArgList overallNps; - std::vector overallLow; - std::vector overallHigh; - std::vector overallInterp; - - RooArgList histNps; - RooArgList histoLo; - RooArgList histoHi; - std::vector histoInterp; -}; - -void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHist &dh) +void buildSample(ChannelContext const &ctx, HS3::Sample const &sample) { RooJSONFactoryWSTool &tool = ctx.tool; RooWorkspace &ws = *tool.workspace(); @@ -602,29 +522,47 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi return name; }; - auto &hf = tool.wsEmplace(newName("hist_" + prefixedName), ctx.observables, dh); + auto dataHist = HS3::createDataHist(prefixedName + "_dataHist", ctx.observables, sample.contents, + sample.errors ? &*sample.errors : nullptr); + auto &hf = tool.wsEmplace(newName("hist_" + prefixedName), ctx.observables, *dataHist); hf.SetTitle(sampleName.c_str()); - SampleElements elems; - elems.shapeElems.add(tool.wsEmplace(newName(prefixedName + "_binWidth"), hf, true)); + // What the modifiers contribute: the factors of the shape and of the normalization of the sample. + RooArgList shapeElems; + RooArgList normElems; + // The arguments of the FlexibleInterpVar for the normsys modifiers. + RooArgList overallNps; + std::vector overallLow; + std::vector overallHigh; + std::vector overallInterp; + // The arguments of the PiecewiseInterpolation for the histosys modifiers. + RooArgList histNps; + RooArgList histoLo; + RooArgList histoHi; + std::vector histoInterp; + std::map histoSysCount; + + shapeElems.add(tool.wsEmplace(newName(prefixedName + "_binWidth"), hf, true)); const bool hasStatError = std::any_of(sample.modifiers.begin(), sample.modifiers.end(), [](auto const &m) { return std::holds_alternative(m); }); if (hasStatError) { - elems.shapeElems.add(*ctx.mcStatObject); + shapeElems.add(*ctx.mcStatObject); } - auto context = [&](const char *type, std::string const &name) { - return std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + sampleName + - "' of channel '" + ctx.channelName + "'"; + // The validation made sure that the channel has a "default_interpolation" if a modifier doesn't give its own. + auto interpolationCode = [&](std::optional const &interpolation, + InterpolationClass interpolationClass, const char *type, std::string const &name) { + return codeFromInterpolation(interpolation ? *interpolation : ctx.dist.defaultInterpolation.value(), + interpolationClass, + std::string("interpolation for ") + type + " modifier '" + name + "' in sample '" + + sampleName + "' of channel '" + ctx.channelName + "'"); }; for (HS3::Modifier const &modifier : sample.modifiers) { std::visit( HS3::overloaded{ - [&](HS3::StatError const &) { - // this is dealt with at a different place, ignore it for now - }, + [](HS3::StatError const &) {}, // built once for the channel, in buildHistFactoryDist() [&](HS3::NormFactor const &mod) { RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 1., -3, 5, mod.name); // Norm factors get an initial zero error like in HistoToWorkspaceFactoryFast, unless an error was @@ -632,98 +570,90 @@ void buildSample(ChannelContext const &ctx, HS3::Sample const &sample, RooDataHi if (auto *var = dynamic_cast(¶m); var && !var->hasError()) { var->setError(0.0); } - elems.normElems.add(param); - // for norm factors, constraints are optional + normElems.add(param); addConstraint(tool, mod.constraint, param, 1.0, "normfactor", mod.name, sampleName, ctx.constraints); }, [&](HS3::NormSys const &mod) { RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); - elems.overallNps.add(param); - const int interp = interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, - InterpolationClass::Flexible, context("normsys", mod.name)); - double low = mod.lo; - double high = mod.hi; - - // the below contains a a hack to cut off variations that go below 0 - // This is needed because FlexibleInterpVar code 4 interpolates in log-space. Hence, values <= 0 result - // in NaN, which propagates throughout the model and causes evaluations to fail. If you know a nicer way - // to solve this, please go ahead and fix the lines below. - if (interp == 4 && low <= 0) - low = std::numeric_limits::epsilon(); - if (interp == 4 && high <= 0) - high = std::numeric_limits::epsilon(); - - elems.overallLow.push_back(low); - elems.overallHigh.push_back(high); - elems.overallInterp.push_back(interp); + overallNps.add(param); + const int interp = + interpolationCode(mod.interpolation, InterpolationClass::Flexible, "normsys", mod.name); + // FlexibleInterpVar code 4 interpolates in log-space, so variations <= 0 would give NaN, which + // propagates through the model and makes its evaluation fail. As a hack, they are cut off. + auto cutOff = [&](double variation) { + return interp == 4 && variation <= 0 ? std::numeric_limits::epsilon() : variation; + }; + overallLow.push_back(cutOff(mod.lo)); + overallHigh.push_back(cutOff(mod.hi)); + overallInterp.push_back(interp); addSysConstraint(tool, mod.constraint, param, "normsys", mod.name, sampleName, ctx.constraints); }, [&](HS3::HistoSys const &mod) { RooAbsReal ¶m = getOrCreateParameter(tool, mod.parameter.name, 0., -5, 5, mod.name); - elems.histNps.add(param); + histNps.add(param); + // Documents before HS3 0.3 can have several histosys modifiers of the same name in a sample (see the + // validation). Their histograms are told apart by a counter. + const int count = ++histoSysCount[mod.name]; + const std::string objName = count == 1 ? mod.name : mod.name + "_" + std::to_string(count); auto addVariation = [&](RooArgList &list, HS3::HistoSysVariation const &variation, const char *suffix) { - const std::string name = newName(mod.name + suffix + prefixedName); + const std::string name = newName(objName + suffix + prefixedName); list.add( tool.wsEmplace(name, ctx.observables, HS3::createDataHist(name, ctx.observables, variation.contents, variation.errors ? &*variation.errors : nullptr))); }; - addVariation(elems.histoLo, mod.lo, "Low_"); - addVariation(elems.histoHi, mod.hi, "High_"); - elems.histoInterp.push_back(interpolationCode(mod.interpolation, ctx.dist.defaultInterpolation, - InterpolationClass::Piecewise, - context("histosys", mod.name))); + addVariation(histoLo, mod.lo, "Low_"); + addVariation(histoHi, mod.hi, "High_"); + histoInterp.push_back( + interpolationCode(mod.interpolation, InterpolationClass::Piecewise, "histosys", mod.name)); addSysConstraint(tool, mod.constraint, param, "histosys", mod.name, sampleName, ctx.constraints); }, [&](HS3::ShapeSys const &mod) { const bool constrained = std::visit( HS3::overloaded{[](HS3::Unconstrained) { return false; }, [](HS3::ConstantGammas) { return false; }, [&](HS3::ConstraintType) { return !mod.vals.empty(); }, - [](HS3::GammaConstraintPdfs const &constraints) { - return std::any_of(constraints.pdfs.begin(), constraints.pdfs.end(), - [](auto const &pdf) { return pdf.has_value(); }); - }}, + [](HS3::GammaConstraintPdfs const &constraints) { return anyNamed(constraints); }}, mod.constraint); // Follow the naming of HistFactory: "__ShapeSys" for constrained and // "__shapeFactor" for free parameters. const std::string funcName = ctx.channelName + "_" + mod.name + (constrained ? "_ShapeSys" : "_shapeFactor"); - elems.shapeElems.add( - createPHF(tool, funcName, mod.name, mod.parameters, mod.vals, mod.constraint, ctx.constraints, - ctx.observables, defaultShapeSysGammaMax, minShapeUncertainty, ctx.nBins, - "the shapesys modifier '" + mod.name + "' of sample '" + sampleName + "'")); + shapeElems.add(createPHF(tool, funcName, mod.name, mod.parameters, mod.vals, mod.constraint, + ctx.constraints, ctx.observables, defaultShapeSysGammaMax, minShapeUncertainty, + sample.contents.size(), + "the shapesys modifier '" + mod.name + "' of sample '" + sampleName + "'")); }, [&](HS3::CustomModifier const &mod) { // The function can come later in the document. RooAbsReal *obj = tool.request(mod.name, ctx.channelName); if (obj->dependsOn(ctx.observables)) { - elems.shapeElems.add(*obj); + shapeElems.add(*obj); } else { - elems.normElems.add(*obj); + normElems.add(*obj); } }, }, modifier); } - if (!elems.overallNps.empty()) { + if (!overallNps.empty()) { std::string interpName = newName(sampleName + "_" + ctx.channelName + "_epsilon"); - elems.normElems.add(tool.wsEmplace( - interpName, elems.overallNps, 1., elems.overallLow, elems.overallHigh, elems.overallInterp)); + normElems.add(tool.wsEmplace(interpName, overallNps, 1., overallLow, + overallHigh, overallInterp)); } - if (!elems.histNps.empty()) { - auto &v = tool.wsEmplace(newName("histoSys_" + prefixedName), hf, elems.histoLo, - elems.histoHi, elems.histNps, elems.histoInterp); + if (!histNps.empty()) { + auto &v = tool.wsEmplace(newName("histoSys_" + prefixedName), hf, histoLo, histoHi, + histNps, histoInterp); v.setPositiveDefinite(); - elems.shapeElems.add(v); + shapeElems.add(v); } else { - elems.shapeElems.add(hf); + shapeElems.add(hf); } - tool.wsEmplace(newName(prefixedName + "_shapes"), elems.shapeElems); - if (!elems.normElems.empty()) { - tool.wsEmplace(newName(prefixedName + "_scaleFactors"), elems.normElems); + tool.wsEmplace(newName(prefixedName + "_shapes"), shapeElems); + if (!normElems.empty()) { + tool.wsEmplace(newName(prefixedName + "_scaleFactors"), normElems); } else { ws.factory("RooConstVar::" + newName(prefixedName + "_scaleFactors") + "(1.)"); } @@ -752,164 +682,136 @@ HS3::GammaConstraint statErrorConstraint(HS3::HistFactoryDist const &dist) return modifierConstraint ? *modifierConstraint : dist.statError.constraint; } -// Reject constraints of staterror modifiers that make no sense for gamma parameters. -void checkStatErrorConstraints(HS3::HistFactoryDist const &dist, std::string const &name) -{ - for (auto const &sample : dist.samples) { - for (auto const &modifier : sample.modifiers) { - auto const *statError = std::get_if(&modifier); - auto const *type = statError ? std::get_if(&statError->constraint) : nullptr; - if (type && *type == HS3::ConstraintType::LogNormal) { - RooJSONFactoryWSTool::error(std::string("unsupported constraint '") + toString(*type) + - "' of the staterror modifier in channel '" + name + - "', supported are \"Poisson\" and \"Gauss\""); - } - } - } -} - +// This is only used for HistFactory distributions that the document doesn't interpret, e.g. because they don't come +// with a document. class HistFactoryImporter : public RooFit::JSONIO::Importer { public: - // This is only used for HistFactory distributions that don't come with a document. - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override - { - RooFit::HS3::importHistFactoryNode(*tool, p, nullptr); - return true; - } -}; - -class FlexibleInterpVarStreamer : public RooFit::JSONIO::Exporter { -public: - std::string const &key() const override + bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &node) const override { - static const std::string keystring = "interpolation0d"; - return keystring; - } - bool exportObject(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem) const override - { - auto fip = static_cast(func); - const std::string context = "FlexibleInterpVar '" + std::string{fip->GetName()} + "'"; - const std::size_t nParameters = fip->variables().size(); - if (fip->low().size() != nParameters || fip->high().size() != nParameters || - fip->interpolationCodes().size() != nParameters) { - RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); - } - HS3::InterpolatedNumber out; - for (RooAbsArg const *var : fip->variables()) { - out.parameters.push_back(var->GetName()); + std::string name = RooJSONFactoryWSTool::name(node); + if (!node.has_child("samples")) { + RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); } - out.nominal = fip->nominal(); - out.high = fip->high(); - out.low = fip->low(); - out.interpolations = toInterpolations(fip->interpolationCodes(), InterpolationClass::Flexible, context); - HS3::writeInterpolatedNumber(elem, out); - return true; - } -}; + // The upgrade rewrites the JSON, so it has to work on a copy. + std::stringstream ss; + node.writeJSON(ss); + std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); -class PiecewiseInterpolationStreamer : public RooFit::JSONIO::Exporter { -public: - std::string const &key() const override - { - static const std::string keystring = "interpolation"; - return keystring; - } - bool exportObject(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem) const override - { - const PiecewiseInterpolation *pip = static_cast(func); - const std::string context = "PiecewiseInterpolation '" + std::string{pip->GetName()} + "'"; - const std::size_t nParameters = pip->paramList().size(); - if (pip->lowList().size() != nParameters || pip->highList().size() != nParameters || - pip->interpolationCodes().size() != nParameters) { - RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); - } - HS3::InterpolatedFunction out; - out.nominal = pip->nominalHist()->GetName(); - for (std::size_t i = 0; i < nParameters; ++i) { - out.parameters.push_back(HS3::toNameOrNumber(pip->paramList()[i])); - out.high.push_back(HS3::toNameOrNumber(pip->highList()[i])); - out.low.push_back(HS3::toNameOrNumber(pip->lowList()[i])); + HS3::Diagnostics diagnostics; + HS3::upgradeHistFactoryDist(tree->rootnode(), tool->hs3Version(), diagnostics); + HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); + if (!HS3::hasErrors(diagnostics)) { + HS3::validateHistFactoryDist(dist, name, diagnostics); } - out.interpolations = toInterpolations(pip->interpolationCodes(), InterpolationClass::Piecewise, context); - out.positiveDefinite = pip->positiveDefinite(); - HS3::writeInterpolatedFunction(elem, out); + HS3::report(diagnostics); + HS3::buildHistFactoryDist(*tool, name, dist); return true; } }; -class PiecewiseInterpolationFactory : public RooFit::JSONIO::Importer { -public: - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override - { - std::string name(RooJSONFactoryWSTool::name(p)); - HS3::Diagnostics diagnostics; - const HS3::InterpolatedFunction in = HS3::parseInterpolatedFunction(p, diagnostics); - report(name, diagnostics); - - auto request = [&](std::vector const &refs) { - RooArgList out; - for (auto const &ref : refs) { - auto const *value = std::get_if(&ref); - out.add(value ? RooFit::RooConst(*value) : *tool->request(std::get(ref), name)); - } - return out; - }; - RooArgList vars{request(in.parameters)}; - RooArgList low{request(in.low)}; - RooArgList high{request(in.high)}; - auto &pip = tool->wsEmplace( - name, *tool->request(in.nominal, name), low, high, vars, - toCodes(in.interpolations, InterpolationClass::Piecewise, "PiecewiseInterpolation '" + name + "'")); - pip.setPositiveDefinite(in.positiveDefinite); - return true; - } -}; +void exportFlexibleInterpVar(RooJSONFactoryWSTool *, RooStats::HistFactory::FlexibleInterpVar const &fip, + JSONNode &elem) +{ + const std::string context = "FlexibleInterpVar '" + std::string{fip.GetName()} + "'"; + const std::size_t nParameters = fip.variables().size(); + if (fip.low().size() != nParameters || fip.high().size() != nParameters || + fip.interpolationCodes().size() != nParameters) { + RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); + } + HS3::InterpolatedNumber out; + for (RooAbsArg const *var : fip.variables()) { + out.parameters.push_back(var->GetName()); + } + out.nominal = fip.nominal(); + out.high = fip.high(); + out.low = fip.low(); + out.interpolations = toInterpolations(fip.interpolationCodes(), InterpolationClass::Flexible, context); + HS3::writeInterpolatedNumber(elem, out); +} -class FlexibleInterpVarFactory : public RooFit::JSONIO::Importer { -public: - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override - { - std::string name(RooJSONFactoryWSTool::name(p)); - HS3::Diagnostics diagnostics; - const HS3::InterpolatedNumber in = HS3::parseInterpolatedNumber(p, diagnostics); - report(name, diagnostics); +void exportPiecewiseInterpolation(RooJSONFactoryWSTool *, PiecewiseInterpolation const &pip, JSONNode &elem) +{ + const std::string context = "PiecewiseInterpolation '" + std::string{pip.GetName()} + "'"; + const std::size_t nParameters = pip.paramList().size(); + if (pip.lowList().size() != nParameters || pip.highList().size() != nParameters || + pip.interpolationCodes().size() != nParameters) { + RooJSONFactoryWSTool::error(context + " has non-matching parameter, variation, and interpolation lengths"); + } + HS3::InterpolatedFunction out; + out.nominal = pip.nominalHist()->GetName(); + for (std::size_t i = 0; i < nParameters; ++i) { + out.parameters.push_back(HS3::toNameOrNumber(pip.paramList()[i])); + out.high.push_back(HS3::toNameOrNumber(pip.highList()[i])); + out.low.push_back(HS3::toNameOrNumber(pip.lowList()[i])); + } + out.interpolations = toInterpolations(pip.interpolationCodes(), InterpolationClass::Piecewise, context); + out.positiveDefinite = pip.positiveDefinite(); + HS3::writeInterpolatedFunction(elem, out); +} - RooArgList vars; - for (std::string const &var : in.parameters) { - vars.add(*tool->request(var, name)); +void importPiecewiseInterpolation(RooJSONFactoryWSTool *tool, const JSONNode &p) +{ + std::string name(RooJSONFactoryWSTool::name(p)); + HS3::Diagnostics diagnostics; + const HS3::InterpolatedFunction in = HS3::parseInterpolatedFunction(p, diagnostics); + HS3::report(diagnostics, name); + + auto request = [&](std::vector const &refs) { + RooArgList out; + for (auto const &ref : refs) { + auto const *value = std::get_if(&ref); + out.add(value ? RooFit::RooConst(*value) : *tool->request(std::get(ref), name)); } - tool->wsEmplace( - name, vars, in.nominal, in.low, in.high, - toCodes(in.interpolations, InterpolationClass::Flexible, "FlexibleInterpVar '" + name + "'")); - return true; + return out; + }; + RooArgList vars{request(in.parameters)}; + RooArgList low{request(in.low)}; + RooArgList high{request(in.high)}; + auto &pip = tool->wsEmplace( + name, *tool->request(in.nominal, name), low, high, vars, + toCodes(in.interpolations, InterpolationClass::Piecewise, "PiecewiseInterpolation '" + name + "'")); + pip.setPositiveDefinite(in.positiveDefinite); +} + +void importFlexibleInterpVar(RooJSONFactoryWSTool *tool, const JSONNode &p) +{ + std::string name(RooJSONFactoryWSTool::name(p)); + HS3::Diagnostics diagnostics; + const HS3::InterpolatedNumber in = HS3::parseInterpolatedNumber(p, diagnostics); + HS3::report(diagnostics, name); + + RooArgList vars; + for (std::string const &var : in.parameters) { + vars.add(*tool->request(var, name)); } -}; + tool->wsEmplace( + name, vars, in.nominal, in.low, in.high, + toCodes(in.interpolations, InterpolationClass::Flexible, "FlexibleInterpVar '" + name + "'")); +} std::string stripOuterParens(const std::string &s) { size_t start = 0; size_t end = s.size(); - while (start < end && s[start] == '(' && s[end - 1] == ')') { + // Whether the parenthesis at `start` is only closed by the one at `end - 1`. + auto enclosesAll = [&]() { int depth = 0; - bool balanced = true; for (size_t i = start; i < end - 1; ++i) { if (s[i] == '(') { ++depth; } else if (s[i] == ')') { --depth; } - if (depth == 0 && i < end - 1) { - balanced = false; - break; + if (depth == 0) { + return false; } } - if (balanced) { - ++start; - --end; - } else { - break; - } + return true; + }; + while (start < end && s[start] == '(' && s[end - 1] == ')' && enclosesAll()) { + ++start; + --end; } return s.substr(start, end - start); } @@ -945,7 +847,6 @@ std::vector splitTopLevelProduct(const std::string &expr) std::vector parts; int depth = 0; size_t start = 0; - bool foundTopLevelStar = false; for (size_t i = 0; i < expr.size(); ++i) { char c = expr[i]; @@ -956,19 +857,16 @@ std::vector splitTopLevelProduct(const std::string &expr) } else if ((c == '+' || c == '-') && depth == 0 && isBinaryOperator(expr, i)) { return {}; // A sum or difference at the top level is not a product, e.g. "1 + 0.2 * k" } else if (c == '*' && depth == 0) { - foundTopLevelStar = true; - std::string sub = expr.substr(start, i - start); - parts.push_back(stripOuterParens(sub)); + parts.push_back(stripOuterParens(expr.substr(start, i - start))); start = i + 1; } } - if (!foundTopLevelStar) { + if (parts.empty()) { return {}; // Not a top-level product } - std::string sub = expr.substr(start); - parts.push_back(stripOuterParens(sub)); + parts.push_back(stripOuterParens(expr.substr(start))); return parts; } @@ -987,51 +885,34 @@ void collectConstraintTerms(RooAbsPdf *pdf, std::vector &terms) // A normsys that is written as the formula "1 +- a * b", where one of a and b is a constrained parameter. struct FormulaNormSys { - RooAbsReal *param = nullptr; - RooAbsPdf *constraint = nullptr; - double variation = 0.0; + RooAbsReal *param; + RooAbsPdf *constraint; + double variation; }; -FormulaNormSys parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula, - std::vector const &channelConstraints) +std::optional parseOverallModifierFormula(const std::string &s, RooFormulaVar *formula, + std::vector const &channelConstraints) { static const std::regex pattern( R"(^\s*1(?:\.0)?\s*([\+\-])\s*([a-zA-Z_][a-zA-Z0-9_]*|[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)\s*\*\s*([a-zA-Z_][a-zA-Z0-9_]*|[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)\s*$)"); - FormulaNormSys sys; std::smatch match; - if (std::regex_match(s, match, pattern)) { - const double sign = match[1].str() == "-" ? -1.0 : 1.0; - - std::string token2 = match[2].str(); - std::string token3 = match[4].str(); - - RooAbsReal *p2 = static_cast(formula->getParameter(token2.c_str())); - RooAbsReal *p3 = static_cast(formula->getParameter(token3.c_str())); - RooRealVar *v2 = dynamic_cast(p2); - RooRealVar *v3 = dynamic_cast(p3); - - auto *constr2 = findConstraint(v2, channelConstraints); - auto *constr3 = findConstraint(v3, channelConstraints); - - if (constr2 && !p3) { - sys.param = p2; - sys.variation = sign * toDouble(token3); - } else if (!p2 && constr3) { - sys.param = p3; - sys.variation = sign * toDouble(token2); - } else if (constr2 && p3 && !constr3) { - sys.param = v2; - sys.variation = sign * p3->getVal(); - } else if (p2 && !constr2 && constr3) { - sys.param = v3; - sys.variation = sign * p2->getVal(); - } - - // A recognized formula has exactly one constrained parameter. - sys.constraint = constr2 ? constr2 : constr3; + if (!std::regex_match(s, match, pattern)) { + return std::nullopt; + } + // The factors a and b are parameters of the formula or numbers. A recognized formula has exactly one constrained + // parameter, and the value of the other factor is the variation. + auto *a = static_cast(formula->getParameter(match[2].str().c_str())); + auto *b = static_cast(formula->getParameter(match[4].str().c_str())); + RooAbsPdf *constraintA = findConstraint(dynamic_cast(a), channelConstraints); + RooAbsPdf *constraintB = findConstraint(dynamic_cast(b), channelConstraints); + if (!constraintA == !constraintB) { + return std::nullopt; } - return sys; + RooAbsReal *other = constraintA ? b : a; + const double sign = match[1].str() == "-" ? -1.0 : 1.0; + const double variation = sign * (other ? other->getVal() : toDouble(match[constraintA ? 4 : 2].str())); + return FormulaNormSys{constraintA ? a : b, constraintA ? constraintA : constraintB, variation}; } void collectElements(RooArgList &elems, RooAbsArg *arg) @@ -1047,12 +928,7 @@ void collectElements(RooArgList &elems, RooAbsArg *arg) bool allRooRealVar(const RooAbsCollection &list) { - for (auto *var : list) { - if (!dynamic_cast(var)) { - return false; - } - } - return true; + return std::all_of(list.begin(), list.end(), [](RooAbsArg *arg) { return dynamic_cast(arg); }); } // The parameters of normsys and histosys modifiers must be variables. @@ -1073,6 +949,8 @@ std::vector contents(RooDataHist const &dataHist) struct ExportedChannel { std::string name; HS3::HistFactoryDist dist; + /// Whether a formula among the factors of a sample was written as several modifiers, see readChannel(). + bool decomposedFormulas = false; // The variables of the first histogram, named after the observables of the channel. std::unique_ptr observables; std::size_t nBins = 0; @@ -1081,22 +959,20 @@ struct ExportedChannel { std::vector customModifiers; /// Parts of formulas that are exported as custom modifiers. They are created for the export. std::vector> temporaries; - // To reconstruct the statistical uncertainties, by bin starting at one. - std::map totYield; - std::map totYield2; - std::map relErrors; + // To reconstruct the statistical uncertainties, by bin. + std::map totYield; + std::map totYield2; + std::map relErrors; // Constant inverse bin widths of HistFactory before ROOT 6.26, which histfactory_dist implies. std::set legacyBinWidthFactors; }; -/// The volume of the first bin and the average bin volume of every histogram of a channel. HistFactory before ROOT -/// 6.26 took the inverse bin width as the number of bins over the range of each observable. -std::vector binVolumes(RooRealSumPdf const &sumpdf) +/// The volume of the first bin and the average bin volume of every histogram among the nodes of a channel. HistFactory +/// before ROOT 6.26 took the inverse bin width as the number of bins over the range of each observable. +std::vector binVolumes(RooArgSet const &channelNodes) { - RooArgSet nodes; - sumpdf.treeNodeServerList(&nodes); std::vector out; - for (RooAbsArg *node : nodes) { + for (RooAbsArg *node : channelNodes) { auto *hf = dynamic_cast(node); if (!hf || hf->dataHist().numEntries() == 0) { continue; @@ -1117,11 +993,10 @@ std::vector binVolumes(RooRealSumPdf const &sumpdf) /// samples with (a constant `RooRealVar` `binWidth__` in the coefficients of the `RooRealSumPdf`). /// `histfactory_dist` implies this factor, so it is not written as a modifier. It is identified by its structure, not /// by its name: a constant that is not floating in any snapshot, used only by the sum or the products of the channel -/// (among the objects of the workspace), -/// with the inverse of the volume of the first bin, or of the average bin volume, of one of the histograms of the -/// channel as value. +/// (among the objects of the workspace), with the inverse of the volume of the first bin, or of the average bin volume, +/// of one of the histograms of the channel as value. bool isLegacyBinWidthFactor(RooAbsArg const &factor, RooRealSumPdf const &sumpdf, RooArgSet const &channelNodes, - std::vector const &volumes, RooWorkspace const *ws) + std::vector const &volumes, RooWorkspace const &ws) { if (!RooJSONFactoryWSTool::config().identifyLegacyBinWidthFactors) { return false; @@ -1130,8 +1005,8 @@ bool isLegacyBinWidthFactor(RooAbsArg const &factor, RooRealSumPdf const &sumpdf if (!dynamic_cast(&factor) && !(var && var->isConstant())) { return false; } - if (var && ws) { - for (auto *snapshot : static_range_cast(ws->getSnapshots())) { + if (var) { + for (auto *snapshot : static_range_cast(ws.getSnapshots())) { auto const *inSnapshot = dynamic_cast(snapshot->find(*var)); if (inSnapshot && !inSnapshot->isConstant()) { return false; @@ -1143,8 +1018,7 @@ bool isLegacyBinWidthFactor(RooAbsArg const &factor, RooRealSumPdf const &sumpdf // observables, so it is not cloned: the original sum and products of the channel are still its clients. auto isOriginalOfChannelNode = [&](RooAbsArg const &client) { const std::string attribute = std::string{"ORIGNAME:"} + client.GetName(); - return sumpdf.getAttribute(attribute.c_str()) || - std::any_of(channelNodes.begin(), channelNodes.end(), + return std::any_of(channelNodes.begin(), channelNodes.end(), [&](RooAbsArg const *node) { return node->getAttribute(attribute.c_str()); }); }; for (RooAbsArg const *client : factor.clients()) { @@ -1154,7 +1028,7 @@ bool isLegacyBinWidthFactor(RooAbsArg const &factor, RooRealSumPdf const &sumpdf } // Objects outside the workspace are no part of the model, e.g. the computation graph of a likelihood that was // created from the model before the export. - if (ws && ws->arg(client->GetName()) != client) { + if (ws.arg(client->GetName()) != client) { continue; } return false; @@ -1170,25 +1044,18 @@ bool isLegacyBinWidthFactor(RooAbsArg const &factor, RooRealSumPdf const &sumpdf /// a shapefactor modifier. Its name doesn't matter. bool isShapeModifierFunction(ParamHistFunc const &phf, ExportedChannel const &channel) { - if (!channel.observables || phf.paramList().size() != channel.nBins || - static_cast(phf.numBins()) != channel.nBins || - phf.dataVars().size() != channel.observables->size()) { - return false; - } - for (RooAbsArg const *obs : phf.dataVars()) { - if (!channel.observables->find(obs->GetName())) { - return false; - } - } - return true; + return channel.observables && phf.paramList().size() == channel.nBins && + static_cast(phf.numBins()) == channel.nBins && phf.dataVars().equals(*channel.observables); } +// Read the channel from the RooRealSumPdf. With `decomposeFormulas`, a RooFormulaVar among the factors of a sample +// that is a product is written as one modifier per factor; otherwise, like any other function, as one custom modifier. ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf, - std::vector const &channelConstraints) + std::vector const &channelConstraints, bool decomposeFormulas = true) { ExportedChannel channel; - RooWorkspace *ws = tool->workspace(); + RooWorkspace const &ws = *tool->workspace(); channel.name = pdfname; erasePrefix(channel.name, "model_"); @@ -1196,22 +1063,21 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna RooArgSet channelNodes; sumpdf->treeNodeServerList(&channelNodes); - const std::vector volumes = binVolumes(*sumpdf); + const std::vector volumes = binVolumes(channelNodes); - auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { - if (!pdf) { - return HS3::Unconstrained{}; - } - channel.constraints.emplace(pdf->GetName(), pdf); - return HS3::ConstraintPdf{{pdf->GetName()}}; - }; - auto gammaConstraint = [&](RooAbsPdf const *pdf) -> std::optional { - if (!pdf) { + // The modifiers name only the constraint terms of the channel. A pdf of a parameter that is not, like an external + // constraint or a prior, stays where it is. + auto constraintTerm = [&](RooAbsPdf const *pdf) -> std::optional { + if (std::find(channelConstraints.begin(), channelConstraints.end(), pdf) == channelConstraints.end()) { return std::nullopt; } channel.constraints.emplace(pdf->GetName(), pdf); return HS3::DistributionRef{pdf->GetName()}; }; + auto constraint = [&](RooAbsPdf const *pdf) -> HS3::Constraint { + auto ref = constraintTerm(pdf); + return ref ? HS3::Constraint{HS3::ConstraintPdf{*ref}} : HS3::Unconstrained{}; + }; for (size_t sampleidx = 0; sampleidx < sumpdf->funcList().size(); ++sampleidx) { PiecewiseInterpolation *pip = nullptr; @@ -1254,12 +1120,7 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna } else { // The bin contents of all samples are written in the order of the axes of the channel (the last axis // varies fastest), which is the order of the observables of the first histogram. - RooArgSet const &vars = *varsPtr; - bool sameOrder = vars.size() == channel.observables->size(); - for (std::size_t i = 0; sameOrder && i < vars.size(); ++i) { - sameOrder = std::string(vars[i]->GetName()) == (*channel.observables)[i]->GetName(); - } - if (!sameOrder) { + if (varsPtr->size() != channel.observables->size() || !varsPtr->hasSameLayout(*channel.observables)) { RooJSONFactoryWSTool::error("the histograms of channel '" + channel.name + "' don't have their observables in the same order, so their bin " "contents can't be written along the same axes"); @@ -1308,7 +1169,7 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna erasePrefix(sysname, "alpha_"); const std::string context = "normsys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; - // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). + // A parameter without constraint is unconstrained. addNormSys(*var, fip->high()[i], fip->low()[i], interpolationFromCode(fip->interpolationCodes()[i], InterpolationClass::Flexible, context), findConstraint(var, channelConstraints)); @@ -1324,37 +1185,35 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna expression.ReplaceAll(("x[" + std::to_string(i) + "]").c_str(), p->GetName()); expression.ReplaceAll(("@" + std::to_string(i)).c_str(), p->GetName()); } - auto components = splitTopLevelProduct(expression.Data()); + std::vector components; + if (decomposeFormulas) { + components = splitTopLevelProduct(expression.Data()); + } + channel.decomposedFormulas |= !components.empty(); if (components.empty()) { // it's not a product, let's just treat it as an unknown element custom.push_back({formula->GetName()}); channel.customModifiers.push_back(formula); } else { - // it is a prododuct, we can try to handle the elements separately + // it is a product, we can try to handle the elements separately std::vector realComponents; int idx = 0; for (auto &comp : components) { - // check if this is a trivial element of a product, we can treat it as its own modifier - auto *part = formula->getParameter(comp.c_str()); - if (part) { + if (auto *part = formula->getParameter(comp.c_str())) { + // a trivial element of a product, which we can treat as its own modifier realComponents.push_back(part); - continue; - } - // check if this is an attempt at explicitly encoding an overallSys - FormulaNormSys sys = parseOverallModifierFormula(comp, formula, channelConstraints); - if (sys.param) { - // Preserve the legacy export behaviour for recognized explicit formulae. - addNormSys(*sys.param, 1.0 + sys.variation, 1.0 - sys.variation, - multiplicativePiecewiseExponential, sys.constraint); - continue; + } else if (auto sys = parseOverallModifierFormula(comp, formula, channelConstraints)) { + // An attempt at explicitly encoding an overallSys. Preserve the legacy export behaviour for + // recognized explicit formulae. + addNormSys(*sys->param, 1.0 + sys->variation, 1.0 - sys->variation, + multiplicativePiecewiseExponential, sys->constraint); + } else { + // this is something non-trivial, let's deal with it separately + std::string name = std::string(formula->GetName()) + "_part" + std::to_string(idx++); + temporaries.push_back({name}); + channel.temporaries.push_back(std::make_unique( + name.c_str(), name.c_str(), comp.c_str(), formula->dependents())); } - - // this is something non-trivial, let's deal with it separately - std::string name = std::string(formula->GetName()) + "_part" + std::to_string(idx); - ++idx; - temporaries.push_back({name}); - channel.temporaries.push_back(std::make_unique( - name.c_str(), name.c_str(), comp.c_str(), formula->dependents())); } self(realComponents, self); } @@ -1370,56 +1229,55 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna collectElements(elems, sumpdf->coefList().at(sampleidx)); processElements(elems, processElements); - // see if we can get the observables if (pip) { + // see if we can get the observables if (auto nh = dynamic_cast(pip->nominalHist())) { updateObservables(*nh); } - } - - if (pip) { for (size_t i = 0; i < pip->paramList().size(); ++i) { + auto lo = dynamic_cast(pip->lowList().at(i)); + auto hi = dynamic_cast(pip->highList().at(i)); + if (!lo || !hi) { + continue; + } RooAbsReal *var = static_cast(pip->paramList().at(i)); std::string sysname(var->GetName()); erasePrefix(sysname, "alpha_"); - if (auto lo = dynamic_cast(pip->lowList().at(i))) { - if (auto hi = dynamic_cast(pip->highList().at(i))) { - // A parameter without constraint is unconstrained, see checkUnconstrainedModifiers(). - const auto *pdf = findConstraint(var, channelConstraints); - checkModifierParameter(*var, pdf); - const std::string context = "histosys modifier '" + sysname + "' in sample '" + sample.name + - "' of channel '" + channel.name + "'"; - histoSys.push_back( - {sysname, - {var->GetName()}, - {contents(hi->dataHist()), std::nullopt}, - {contents(lo->dataHist()), std::nullopt}, - constraint(pdf), - interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context)}); - } - } + // A parameter without constraint is unconstrained. + const auto *pdf = findConstraint(var, channelConstraints); + checkModifierParameter(*var, pdf); + const std::string context = + "histosys modifier '" + sysname + "' in sample '" + sample.name + "' of channel '" + channel.name + "'"; + histoSys.push_back( + {sysname, + {var->GetName()}, + {contents(hi->dataHist()), std::nullopt}, + {contents(lo->dataHist()), std::nullopt}, + constraint(pdf), + interpolationFromCode(pip->interpolationCodes()[i], InterpolationClass::Piecewise, context)}); } } for (ParamHistFunc *phf : phfs) { - if (startsWith(std::string(phf->GetName()), "mc_stat_")) { // MC stat uncertainty + if (startsWith(phf->GetName(), "mc_stat_")) { // MC stat uncertainty statError.emplace(); statError->name = "staterror"; HS3::GammaConstraintPdfs pdfs; - int idx = 0; // In the order of the bins (last axis fastest), like the contents of the samples. for (RooAbsReal *g : paramHistFuncParamsInBinOrder(*phf)) { + const std::size_t bin = statError->parameters.size(); statError->parameters.push_back({g->GetName()}); - ++idx; RooAbsPdf *pdf = findConstraint(g, channelConstraints); - pdfs.pdfs.push_back(gammaConstraint(pdf)); - channel.totYield[idx] += sample.contents[idx - 1]; - channel.totYield2[idx] += (sample.contents[idx - 1] * sample.contents[idx - 1]); + pdfs.pdfs.push_back(constraintTerm(pdf)); + channel.totYield[bin] += sample.contents[bin]; + channel.totYield2[bin] += sample.contents[bin] * sample.contents[bin]; if (pdf) { - channel.relErrors[idx] = constraintRelError(*pdf, *g); + channel.relErrors[bin] = constraintRelError(*pdf, *g); } } - statError->constraint = std::move(pdfs); + if (anyNamed(pdfs)) { + statError->constraint = std::move(pdfs); + } } else { // ShapeSys or ShapeFactor // Other ParamHistFuncs can't be written as shapefactors: the import would recreate the function over the // observables and bins of the channel. Over other variables, they are written as custom modifiers, which @@ -1450,10 +1308,12 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna sys.parameters.push_back({g->GetName()}); RooAbsPdf *pdf = findConstraint(g, channelConstraints); if (!pdf) - pdf = ws->pdf(constraintName(g->GetName())); - pdfs.pdfs.push_back(gammaConstraint(pdf)); + pdf = ws.pdf(constraintName(g->GetName())); + pdfs.pdfs.push_back(constraintTerm(pdf)); + } + if (anyNamed(pdfs)) { + sys.constraint = std::move(pdfs); } - sys.constraint = std::move(pdfs); } } @@ -1482,6 +1342,23 @@ ExportedChannel readChannel(RooJSONFactoryWSTool *tool, const std::string &pdfna return channel; } +// Whether the distribution of the channel passes the checks of a HistFactory distribution, with its duplicate +// modifiers combined like for the export. +bool isValidChannel(ExportedChannel const &channel, std::string const &pdfname) +{ + if (!channel.observables) { + return true; // nothing was read that could be checked + } + HS3::HistFactoryDist dist = channel.dist; + HS3::Diagnostics diagnostics; + HS3::mergeDuplicateModifiers(dist, channel.name, diagnostics); + for (auto *obs : static_range_cast(*channel.observables)) { + dist.axes.push_back(HS3::toBinnedAxis(*obs)); + } + HS3::validateHistFactoryDist(dist, pdfname, diagnostics); + return !HS3::hasErrors(diagnostics); +} + // The statistical uncertainties of the samples are reconstructed from the constraints of the gamma parameters, with // the (arbitrary) convention that all samples have the same relative uncertainty. void configureStatError(ExportedChannel &channel) @@ -1493,31 +1370,34 @@ void configureStatError(ExportedChannel &channel) } std::vector errors(sample.contents.size()); for (auto const &bin : channel.relErrors) { - const int i = bin.first; + const std::size_t i = bin.first; // this reconstruction is inherently imprecise, so we truncate it at some decimal places to make sure that // we don't carry around too many useless digits - errors[i - 1] = - round_prec(bin.second * channel.totYield[i] / std::sqrt(channel.totYield2[i]) * sample.contents[i - 1], 7); + errors[i] = + round_prec(bin.second * channel.totYield[i] / std::sqrt(channel.totYield2[i]) * sample.contents[i], 7); } sample.errors = std::move(errors); } } -// Whether `pdf` has the form that the HistFactory exporter writes as a histfactory_dist with its constraint terms: -// a product of exactly one RooRealSumPdf and other pdfs. -bool isHistFactoryChannelProduct(RooAbsArg const &pdf) +// The RooRealSumPdf of a product of exactly one RooRealSumPdf and other pdfs, or null for anything else. Such a product +// has the form that the HistFactory exporter writes as a histfactory_dist with its constraint terms. +RooRealSumPdf const *channelSum(RooAbsArg const &pdf) { auto const *prod = dynamic_cast(&pdf); if (!prod) { - return false; + return nullptr; } - int nSums = 0; + RooRealSumPdf const *sum = nullptr; for (RooAbsArg const *factor : prod->pdfList()) { - if (dynamic_cast(factor)) { - ++nSums; + if (auto const *factorSum = dynamic_cast(factor)) { + if (sum) { + return nullptr; + } + sum = factorSum; } } - return nSums == 1; + return sum; } // Whether the constraint `constraint` can be written as an array component: every object of the workspace that uses @@ -1527,17 +1407,23 @@ bool isHistFactoryChannelProduct(RooAbsArg const &pdf) // which the export visits the objects. bool usedOnlyByHistFactoryChannels(RooJSONFactoryWSTool &tool, RooAbsPdf const &constraint) { - for (TObject const *obj : tool.workspace()->allGenericObjects()) { - auto const *mc = dynamic_cast(obj); - if (mc && mc->GetExternalConstraints() && mc->GetExternalConstraints()->find(constraint.GetName())) { - return false; + // A component of an array distribution stays in it (see exportConstraint()); a ModelConfig refers to it through + // the array. + const bool inArray = std::any_of(constraint.clients().begin(), constraint.clients().end(), + [](RooAbsArg const *client) { return client->getAttribute(arrayDistTag); }); + if (!inArray) { + for (TObject const *obj : tool.workspace()->allGenericObjects()) { + auto const *mc = dynamic_cast(obj); + if (mc && mc->GetExternalConstraints() && mc->GetExternalConstraints()->find(constraint.GetName())) { + return false; + } } } for (RooAbsArg const *client : constraint.clients()) { - if (tool.workspace()->arg(client->GetName()) != client) { + if (tool.workspace()->arg(client->GetName()) != client || isSkippedHelper(*client, *tool.workspace())) { continue; // not part of what is exported } - if (!client->getAttribute(arrayDistTag) && !isHistFactoryChannelProduct(*client)) { + if (!client->getAttribute(arrayDistTag) && !channelSum(*client)) { return false; } } @@ -1553,8 +1439,9 @@ std::string exportConstraint(RooJSONFactoryWSTool &tool, RooAbsPdf const &constr if (RooJSONFactoryWSTool::config().exportArrayDist && usedOnlyByHistFactoryChannels(tool, constraint)) { if (auto info = arrayComponentInfo(constraint)) { std::string arrayName; + // An array that is written: one that only skipped helpers use is not (see isSkippedHelper()). for (RooAbsArg const *client : constraint.clients()) { - if (client->getAttribute(arrayDistTag)) { + if (client->getAttribute(arrayDistTag) && !isSkippedHelper(*client, *tool.workspace())) { arrayName = client->GetName(); break; } @@ -1608,7 +1495,7 @@ std::vector exportNamedConstraints(RooJSONFactoryWSTool &tool, Expo if (auto *pdfs = std::get_if(&constraint)) { for (std::size_t i = 0; i < pdfs->pdfs.size(); ++i) { if (auto &pdf = pdfs->pdfs[i]) { - rename(pdf->name, i < params.size() ? params[i].name : std::string{}); + rename(pdf->name, params[i].name); } } } @@ -1627,56 +1514,6 @@ std::vector exportNamedConstraints(RooJSONFactoryWSTool &tool, Expo return terms; } -// The modifiers name only the constraint terms of the channel. A pdf of a parameter that is not, like an external -// constraint or a prior, stays where it is. Returns the named constraint terms. -std::set -nameChannelConstraints(ExportedChannel &channel, std::vector const &constraints) -{ - std::set named; - auto isNamed = [&](std::string const &name) { - RooAbsPdf const *pdf = channel.constraints.at(name); - if (std::find(constraints.begin(), constraints.end(), pdf) == constraints.end()) { - return false; - } - named.insert(pdf); - return true; - }; - auto filter = [&](HS3::Constraint &constraint) { - auto const *pdf = std::get_if(&constraint); - if (pdf && !isNamed(pdf->pdf.name)) { - constraint = HS3::Unconstrained{}; - } - }; - auto filterGammas = [&](HS3::GammaConstraint &constraint) { - auto *pdfs = std::get_if(&constraint); - if (!pdfs) { - return; - } - bool any = false; - for (auto &pdf : pdfs->pdfs) { - if (pdf && !isNamed(pdf->name)) { - pdf.reset(); - } - any = any || pdf; - } - // Nothing is written if all parameters are unconstrained. - if (!any) { - constraint = HS3::Unconstrained{}; - } - }; - for (HS3::Sample &sample : channel.dist.samples) { - for (HS3::Modifier &modifier : sample.modifiers) { - std::visit(HS3::overloaded{ - [&](HS3::NormFactor &m) { filter(m.constraint); }, - [&](HS3::NormSys &m) { filter(m.constraint); }, [&](HS3::HistoSys &m) { filter(m.constraint); }, - [&](HS3::ShapeSys &m) { filterGammas(m.constraint); }, - [&](HS3::StatError &m) { filterGammas(m.constraint); }, [](HS3::CustomModifier &) {}}, - modifier); - } - } - return named; -} - bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname, const RooRealSumPdf *sumpdf, std::vector constraints) { @@ -1692,6 +1529,15 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname } ExportedChannel channel = readChannel(tool, pdfname, sumpdf, constraints); + // Writing a formula as several modifiers must not be what makes the channel invalid, e.g. because a parameter that + // occurs twice in the formula becomes two normfactors of the same name. If it is, the formulas are written as + // custom modifiers instead. + if (channel.decomposedFormulas && !isValidChannel(channel, pdfname)) { + ExportedChannel plain = readChannel(tool, pdfname, sumpdf, constraints, /*decomposeFormulas=*/false); + if (isValidChannel(plain, pdfname)) { + channel = std::move(plain); + } + } HS3::HistFactoryDist &dist = channel.dist; // sanity checks @@ -1713,7 +1559,6 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname dist.axes.push_back(HS3::toBinnedAxis(*obs)); } - std::set namedConstraints = nameChannelConstraints(channel, constraints); tool->recordConstraintTerms(pdfname, exportNamedConstraints(*tool, channel)); for (RooAbsReal *modifier : channel.customModifiers) { @@ -1728,7 +1573,7 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname sumpdf->getParameters(channel.observables.get(), parameters); for (RooAbsArg *param : parameters) { // This should exclude the global observables. The legacy inverse bin widths are not written at all. - if (!startsWith(std::string{param->GetName()}, "nom_") && !channel.legacyBinWidthFactors.count(param)) { + if (!startsWith(param->GetName(), "nom_") && !channel.legacyBinWidthFactors.count(param)) { tool->queueExport(*param); } } @@ -1736,7 +1581,7 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname // All other constraint terms of the channel are written separately. std::vector channelConstraintNames; for (RooAbsPdf const *pdf : constraints) { - if (!namedConstraints.count(pdf)) { + if (!channel.constraints.count(pdf->GetName())) { tool->queueExport(*pdf); channelConstraintNames.push_back({pdf->GetName()}); } @@ -1746,31 +1591,10 @@ bool tryExportHistFactory(RooJSONFactoryWSTool *tool, const std::string &pdfname return true; } -class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { +// Writes a RooRealSumPdf, or the product of one with its constraint terms, as a histfactory_dist. +class HistFactoryStreamer : public RooFit::JSONIO::Exporter { public: bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooProdPdf *prodpdf) const - { - std::vector constraints; - RooRealSumPdf *sumpdf = nullptr; - for (auto *pdf : static_range_cast(prodpdf->pdfList())) { - auto thispdf = dynamic_cast(pdf); - if (thispdf) { - if (!sumpdf) { - sumpdf = thispdf; - } else { - return false; - } - } else { - // The constraint terms can also be grouped in nested products. - collectConstraintTerms(pdf, constraints); - } - } - if (!sumpdf) - return false; - - return tryExportHistFactory(tool, prodpdf->GetName(), sumpdf, constraints); - } std::string const &key() const override { static const std::string keystring = "histfactory_dist"; @@ -1778,145 +1602,122 @@ class HistFactoryStreamer_ProdPdf : public RooFit::JSONIO::Exporter { } bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override { - return tryExport(tool, static_cast(p)); - } -}; - -class HistFactoryStreamer_SumPdf : public RooFit::JSONIO::Exporter { -public: - bool autoExportDependants() const override { return false; } - bool tryExport(RooJSONFactoryWSTool *tool, const RooRealSumPdf *sumpdf) const - { + auto const *sumpdf = dynamic_cast(p); std::vector constraints; - return tryExportHistFactory(tool, sumpdf->GetName(), sumpdf, constraints); - } - std::string const &key() const override - { - static const std::string keystring = "histfactory_dist"; - return keystring; - } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *p, JSONNode &) const override - { - return tryExport(tool, static_cast(p)); + if (auto const *prodpdf = dynamic_cast(p)) { + sumpdf = channelSum(*prodpdf); + for (auto *pdf : static_range_cast(prodpdf->pdfList())) { + if (pdf != sumpdf) { + // The constraint terms can also be grouped in nested products. + collectConstraintTerms(pdf, constraints); + } + } + } + return tryExportHistFactory(tool, p->GetName(), sumpdf, constraints); } }; -STATIC_EXECUTE([]() { - using namespace RooFit::JSONIO; - - registerImporter("histfactory_dist", true); - registerImporter("interpolation", true); - registerImporter("interpolation0d", true); - registerExporter(RooStats::HistFactory::FlexibleInterpVar::Class(), true); - registerExporter(PiecewiseInterpolation::Class(), true); - registerExporter(RooProdPdf::Class(), true); - registerExporter(RooRealSumPdf::Class(), true); -}); +// Registers the importers and exporters when the library is loaded. +const bool registered = []() { + registerType("interpolation", importPiecewiseInterpolation, exportPiecewiseInterpolation); + registerType("interpolation0d", importFlexibleInterpVar, exportFlexibleInterpVar); + RooFit::JSONIO::registerImporter("histfactory_dist", true); + RooFit::JSONIO::registerExporter(RooProdPdf::Class(), true); + RooFit::JSONIO::registerExporter(RooRealSumPdf::Class(), true); + return true; +}(); } // namespace -void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist, - std::vector const *channelConstraints) +void RooFit::HS3::buildHistFactoryDist(RooJSONFactoryWSTool &tool, std::string const &name, HistFactoryDist const &dist) { - checkStatErrorConstraints(dist, name); - RooArgSet observables = createObservables(dist.axes); - std::vector> data; - data.reserve(dist.samples.size()); + // The yields and their variances by bin, summed over the samples with a staterror modifier. std::vector sumW; std::vector sumW2; std::vector gammaParnames; for (auto const &sample : dist.samples) { - auto dh = createDataHist(name + "_" + sample.name + "_dataHist", observables, sample.contents, - sample.errors ? &*sample.errors : nullptr); - const std::size_t nbins = dh->numEntries(); - auto found = std::find_if(sample.modifiers.begin(), sample.modifiers.end(), [](auto const &m) { return std::holds_alternative(m); }); - if (found != sample.modifiers.end()) { - if (sumW.empty()) { - sumW.resize(nbins); - sumW2.resize(nbins); - } + if (found == sample.modifiers.end()) { + continue; + } + sumW.resize(sample.contents.size()); + sumW2.resize(sample.contents.size()); + for (std::size_t i = 0; i < sample.contents.size(); ++i) { + sumW[i] += sample.contents[i]; // A sample without "errors" contributes no statistical uncertainty, unless Poisson uncertainties are implied. - const bool hasErrors = sample.errors || dist.statError.impliedPoissonErrors; - for (std::size_t i = 0; i < nbins; ++i) { - sumW[i] += dh->weight(i); - sumW2[i] += hasErrors ? dh->weightSquared(i) : 0.; - } - if (gammaParnames.empty()) { - gammaParnames = std::get(*found).parameters; + if (sample.errors) { + sumW2[i] += (*sample.errors)[i] * (*sample.errors)[i]; + } else if (dist.statError.impliedPoissonErrors) { + sumW2[i] += sample.contents[i]; } } - data.emplace_back(std::move(dh)); + if (gammaParnames.empty()) { + gammaParnames = std::get(*found).parameters; + } } std::string channelName = name; erasePrefix(channelName, "model_"); - std::size_t nBins = 1; - for (auto const &axis : dist.axes) { - nBins *= std::visit(overloaded{[](RegularBinning const &b) { return static_cast(b.nbins); }, - [](EdgesBinning const &b) { return b.edges.size() - 1; }}, - axis.binning); - } - RooAbsArg *mcStatObject = nullptr; RooArgSet constraints; if (!sumW.empty()) { + // The relative uncertainties. The gammas of bins without a positive yield are set constant later. std::vector errs(sumW.size()); for (std::size_t i = 0; i < sumW.size(); ++i) { - if (sumW[i] == 0.) { - errs[i] = 0.; - continue; - } - errs[i] = std::sqrt(sumW2[i]) / sumW[i]; - // avoid negative sigma. This NP will be set constant anyway later - errs[i] = std::max(errs[i], 0.); + errs[i] = sumW[i] > 0. ? std::sqrt(sumW2[i]) / sumW[i] : 0.; } + // In HS3 0.3 (HS3 PR #154, decision on HS3 #156), the named constraints of a staterror are authoritative: the + // range, the constness and the step width of the gammas come from the document, and nothing is derived from + // the errors of the samples. Older documents relied on RooFit configuring the gammas from them. + const bool configureGammas = tool.hs3Version() < HS3::Version{0, 3}; mcStatObject = &createPHF(tool, "mc_stat_" + channelName, "stat_" + channelName, gammaParnames, errs, statErrorConstraint(dist), constraints, observables, defaultStatErrorGammaMax, - dist.statError.relThreshold, nBins, "the staterror modifier of channel '" + name + "'"); + dist.statError.relThreshold, errs.size(), + "the staterror modifier of channel '" + name + "'", configureGammas); } - ChannelContext ctx{tool, dist, observables, name, channelName, nBins, mcStatObject, constraints}; + ChannelContext ctx{tool, dist, observables, name, channelName, mcStatObject, constraints}; RooArgList funcs; RooArgList coefs; - for (std::size_t i = 0; i < dist.samples.size(); ++i) { - HS3::Sample const &sample = dist.samples[i]; - buildSample(ctx, sample, *data[i]); + for (HS3::Sample const &sample : dist.samples) { + buildSample(ctx, sample); funcs.add(*tool.request(name + "_" + sample.name + "_shapes", name)); coefs.add(*tool.request(name + "_" + sample.name + "_scaleFactors", name)); } - if (channelConstraints) { + if (auto const *channelConstraints = tool.channelConstraints(name)) { for (auto const &ref : *channelConstraints) { - RooAbsPdf *pdf = tool.tryRequest(ref.name); - if (!pdf) { - RooJSONFactoryWSTool::error("unable to find the constraint '" + ref.name + "' of '" + name + "'"); + constraints.add(*tool.request(ref.name, name)); + } + } + + // A constraint that is a measurement of its own, as a component of a combined distribution with data, contributes + // there, and is not multiplied into the channel. + { + RooArgSet ownMeasurements; + for (auto *constraint : static_range_cast(constraints)) { + if (tool.isConstraintReadAsChannel(*constraint)) { + ownMeasurements.add(*constraint); } - constraints.add(*pdf); } + constraints.remove(ownMeasurements); } + // With constraints, the channel is the product of the sum of the samples and the constraints. + auto &sum = tool.wsEmplace(constraints.empty() ? name : channelName + "_model", funcs, coefs, true); + sum.SetTitle(name.c_str()); // A histfactory_dist is a product of Poisson distributions of the bin counts. RooFit's binned likelihood // computes exactly that (it differs from the event-wise likelihood only by a constant), and binned generation // generates bin counts, so both are enabled, as HistFactory does by default. - auto enableBinned = [](RooRealSumPdf &sum) { - sum.setAttribute("BinnedLikelihood"); - sum.setAttribute("GenerateBinned"); - }; - if (constraints.empty()) { - enableBinned(tool.wsEmplace(name, funcs, coefs, true)); - } else { - std::string sumName = name + "_model"; - erasePrefix(sumName, "model_"); - auto &sum = tool.wsEmplace(sumName, funcs, coefs, true); - sum.SetTitle(name.c_str()); - enableBinned(sum); + sum.setAttribute("BinnedLikelihood"); + sum.setAttribute("GenerateBinned"); + if (!constraints.empty()) { tool.wsEmplace(name, constraints, RooFit::Conditional(sum, observables)); } } @@ -1925,28 +1726,3 @@ bool RooFit::HS3::isBuiltinHistFactoryImporter(RooFit::JSONIO::Importer const &i { return dynamic_cast(&importer) != nullptr; } - -// The upgrade rewrites the JSON, so it has to work on a copy. -void RooFit::HS3::importHistFactoryNode(RooJSONFactoryWSTool &tool, JSONNode const &p, - std::vector const *channelConstraints) -{ - std::string name = RooJSONFactoryWSTool::name(p); - if (!p.has_child("samples")) { - RooJSONFactoryWSTool::error("no samples in '" + name + "', skipping."); - } - std::stringstream ss; - p.writeJSON(ss); - std::unique_ptr tree = RooFit::Detail::JSONTree::create(ss.str()); - - HS3::Version version; - HS3::parseVersion(tool.hs3VersionInput(), version); - - HS3::Diagnostics diagnostics; - HS3::upgradeHistFactoryDist(tree->rootnode(), version, diagnostics); - HS3::HistFactoryDist dist = HS3::parseHistFactoryDist(tree->rootnode(), diagnostics); - if (!HS3::hasErrors(diagnostics)) { - HS3::validateHistFactoryDist(dist, name, diagnostics); - } - HS3::report(diagnostics); - HS3::buildHistFactoryDist(tool, name, dist, channelConstraints); -} diff --git a/roofit/hs3/src/JSONFactories_RooFitCore.cxx b/roofit/hs3/src/JSONFactories_RooFitCore.cxx index eb01d2e04cec8..f0a91f0a7df18 100644 --- a/roofit/hs3/src/JSONFactories_RooFitCore.cxx +++ b/roofit/hs3/src/JSONFactories_RooFitCore.cxx @@ -17,12 +17,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -64,35 +62,29 @@ #include #include #include -#include #include #include -#include -#include -#include +#include "HS3Builder.h" #include "JSONIOUtils.h" #include #include #include -#include "static_execute.h" - #include #include -#include -#include #include #include #include -#include #include #include using RooFit::Detail::JSONNode; +namespace HS3 = RooFit::HS3; + /////////////////////////////////////////////////////////////////////////////////////////////////////// // individually implemented importers /////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -103,16 +95,20 @@ bool isReservedExpressionIdentifier(const std::string &arg) return arg == "PI" || arg == "EULER" || arg == "TMath"; } +bool isIdentifierChar(char c) +{ + return std::isalnum(static_cast(c)) || c == '_'; +} + /** * Extracts arguments from a mathematical expression. * * This function takes a string representing a mathematical * expression and extracts the arguments from it. The arguments are - * defined as sequences of characters that do not contain digits, - * spaces, or parentheses, and that start with a letter. Function - * calls such as "exp( ... )", identified as being followed by an - * opening parenthesis, are not treated as arguments. The extracted - * arguments are returned as a vector of strings. + * the identifiers: sequences of letters, digits and underscores that + * start with a letter. Function calls such as "exp( ... )", identified + * as being followed by an opening parenthesis, are not treated as + * arguments, and neither are the reserved identifiers. * * @param expr A string representing a mathematical expression. * @return A set of unique strings representing the extracted arguments. @@ -123,34 +119,18 @@ std::set extractArguments(std::string expr) expr.erase(std::remove_if(expr.begin(), expr.end(), [](unsigned char c) { return std::isspace(c); }), expr.end()); std::set arguments; - size_t startidx = expr.size(); - for (size_t i = 0; i < expr.size(); ++i) { - if (startidx >= expr.size()) { - if (isalpha(expr[i])) { - startidx = i; - // check this character is not part of scientific notation, e.g. 2e-5 - if (TFormula::IsScientificNotation(expr, i)) { - // if it is, we ignore this character - startidx = expr.size(); - } - } - } else { - if (!isdigit(expr[i]) && !isalpha(expr[i]) && expr[i] != '_') { - if (expr[i] == '(') { - startidx = expr.size(); - continue; - } - std::string arg(expr.substr(startidx, i - startidx)); - startidx = expr.size(); - if (!isReservedExpressionIdentifier(arg)) { - arguments.insert(arg); - } - } + for (std::size_t i = 0; i < expr.size(); ++i) { + // A letter starts an identifier, unless it is part of a number in scientific notation, e.g. 2e-5. + if (!isalpha(expr[i]) || TFormula::IsScientificNotation(expr, i)) { + continue; } - } - if (startidx < expr.size()) { - std::string arg(expr.substr(startidx)); - if (!isReservedExpressionIdentifier(arg)) { + const std::size_t start = i; + while (i < expr.size() && isIdentifierChar(expr[i])) { + ++i; + } + const std::string arg = expr.substr(start, i - start); + const bool isFunctionCall = i < expr.size() && expr[i] == '('; + if (!isFunctionCall && !isReservedExpressionIdentifier(arg)) { arguments.insert(arg); } } @@ -165,11 +145,9 @@ void replaceIdentifier(TString &expr, std::string_view identifier, std::string_v for (std::size_t pos = 0; pos < in.size();) { const bool matches = in.compare(pos, identifier.size(), identifier) == 0; - const bool beforeIdentifier = - pos > 0 && (std::isalnum(static_cast(in[pos - 1])) || in[pos - 1] == '_'); + const bool beforeIdentifier = pos > 0 && isIdentifierChar(in[pos - 1]); const std::size_t end = pos + identifier.size(); - const bool afterIdentifier = - end < in.size() && (std::isalnum(static_cast(in[end])) || in[end] == '_'); + const bool afterIdentifier = end < in.size() && isIdentifierChar(in[end]); if (matches && !beforeIdentifier && !afterIdentifier) { out.append(replacement); pos = end; @@ -182,11 +160,6 @@ void replaceIdentifier(TString &expr, std::string_view identifier, std::string_v expr = out.c_str(); } -bool isIdentifierChar(char c) -{ - return std::isalnum(static_cast(c)) || c == '_'; -} - std::string trimmed(std::string const &str) { const auto begin = str.find_first_not_of(" \t\n"); @@ -210,22 +183,34 @@ std::size_t findClosingBracket(std::string const &str, std::size_t open) return std::string::npos; } -// Split `str` at the commas that are not enclosed in brackets. -std::vector splitTopLevelCommas(std::string const &str) +// Index of the first of the characters `chars` in `str` from `start` on that is not enclosed in brackets, or +// std::string::npos if there is none. The scope operator "::" is skipped: a ':' is the one of a ternary operator. +std::size_t findTopLevel(std::string const &str, std::string_view chars, std::size_t start = 0) { - std::vector parts; int depth = 0; - std::size_t start = 0; - for (std::size_t i = 0; i < str.size(); ++i) { + for (std::size_t i = start; i < str.size(); ++i) { if (str[i] == '(' || str[i] == '[') { ++depth; } else if (str[i] == ')' || str[i] == ']') { --depth; - } else if (str[i] == ',' && depth == 0) { - parts.push_back(str.substr(start, i - start)); - start = i + 1; + } else if (str[i] == ':' && i + 1 < str.size() && str[i + 1] == ':') { + ++i; + } else if (depth == 0 && chars.find(str[i]) != std::string_view::npos) { + return i; } } + return std::string::npos; +} + +// Split `str` at the commas that are not enclosed in brackets. +std::vector splitTopLevelCommas(std::string const &str) +{ + std::vector parts; + std::size_t start = 0; + for (std::size_t comma = findTopLevel(str, ","); comma != std::string::npos; comma = findTopLevel(str, ",", start)) { + parts.push_back(str.substr(start, comma - start)); + start = comma + 1; + } parts.push_back(str.substr(start)); return parts; } @@ -275,122 +260,49 @@ void translateImportedExpression(TString &expr) { replaceIdentifier(expr, "PI", "TMath::Pi()"); replaceIdentifier(expr, "EULER", "TMath::E()"); - if (std::string(expr.Data()).find("ifelse") != std::string::npos) { - expr = ifElseToTernary(expr.Data()).c_str(); - } -} - -int readPositiveInteger(const JSONNode &node, const std::string &context) -{ - // Read through val_double() so an integer encoded as a JSON float (e.g. 1e6, - // whose textual form is "1e+06") is accepted like elsewhere in HS3, while - // fractional, non-finite, out-of-range or non-numeric values are rejected. - const double value = node.is_number() ? node.val_double() : std::numeric_limits::quiet_NaN(); - if (!std::isfinite(value) || value < 1.0 || value != std::floor(value) || - value > static_cast(std::numeric_limits::max())) { - RooJSONFactoryWSTool::error("\"nbins\" in " + context + " must be a positive integer"); - } - return static_cast(value); -} - -std::unique_ptr -readFormulaAxisBinning(const JSONNode &axis, const std::string &axisName, const std::string &formulaName) -{ - const std::string context = "axis '" + axisName + "' of generic formula '" + formulaName + "'"; - const bool hasEdges = axis.has_child("edges"); - const bool hasMin = axis.has_child("min"); - const bool hasMax = axis.has_child("max"); - const bool hasNBins = axis.has_child("nbins"); - - if (hasEdges && hasNBins) { - RooJSONFactoryWSTool::error(context + " must use either \"edges\" or \"nbins\""); - } - - if (hasEdges) { - const JSONNode &edgesNode = axis["edges"]; - if (!edgesNode.is_seq()) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must be a sequence"); - } - - std::vector edges; - edges.reserve(edgesNode.num_children()); - for (const JSONNode &edgeNode : edgesNode.children()) { - if (!edgeNode.is_number()) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain only finite values"); - } - const double edge = edgeNode.val_double(); - if (!std::isfinite(edge)) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain only finite values"); - } - if (!edges.empty() && edge <= edges.back()) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must be strictly increasing"); - } - edges.push_back(edge); - } - if (edges.size() < 2) { - RooJSONFactoryWSTool::error("\"edges\" in " + context + " must contain at least two values"); - } - // The HS3 standard requires "min" and "max" also for irregular binnings, which older versions of RooFit didn't - // write. - if ((hasMin && (!axis["min"].is_number() || axis["min"].val_double() != edges.front())) || - (hasMax && (!axis["max"].is_number() || axis["max"].val_double() != edges.back()))) { - RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must agree with the \"edges\""); - } - return std::make_unique(static_cast(edges.size() - 1), edges.data()); - } - - if (!hasMin || !hasMax || !hasNBins) { - RooJSONFactoryWSTool::error(context + " must define \"min\", \"max\", and \"nbins\""); - } - - if (!axis["min"].is_number() || !axis["max"].is_number()) { - RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must be finite and increasing"); - } - const double min = axis["min"].val_double(); - const double max = axis["max"].val_double(); - if (!std::isfinite(min) || !std::isfinite(max) || max <= min) { - RooJSONFactoryWSTool::error("\"min\" and \"max\" in " + context + " must be finite and increasing"); - } - return std::make_unique(min, max, readPositiveInteger(axis["nbins"], context)); + // The boolean literals, and the exact comparisons, which are the ones of TFormula. + replaceIdentifier(expr, "TRUE", "1"); + replaceIdentifier(expr, "FALSE", "0"); + expr.ReplaceAll("!==", "!="); + expr.ReplaceAll("===", "=="); + expr = ifElseToTernary(expr.Data()).c_str(); } +// Give the formula the binnings of its "axes", which are read like the axes of binned data. template void importFormulaBinnings(RooArg_t &arg, const JSONNode &node) { if (!node.has_child("axes")) { return; } - - const JSONNode &axes = node["axes"]; - if (!axes.is_seq()) { - RooJSONFactoryWSTool::error("\"axes\" in generic formula '" + std::string(arg.GetName()) + - "' must be a sequence"); - } + const std::string context = "generic formula '" + std::string(arg.GetName()) + "'"; + HS3::Diagnostics diagnostics; + const std::vector axes = HS3::parseBinnedAxes(node["axes"], diagnostics); + HS3::report(diagnostics, std::string(arg.GetName()) + ".axes"); std::set axisNames; - for (const JSONNode &axis : axes.children()) { - if (!axis.is_map() || !axis.has_child("name")) { - RooJSONFactoryWSTool::error("each axis in generic formula '" + std::string(arg.GetName()) + - "' must be a map with a \"name\""); + for (HS3::BinnedAxis const &axis : axes) { + if (!axisNames.insert(axis.name).second) { + RooJSONFactoryWSTool::error("duplicate axis '" + axis.name + "' in " + context); } - const std::string axisName = axis["name"].val(); - if (!axisNames.insert(axisName).second) { - RooJSONFactoryWSTool::error("duplicate axis '" + axisName + "' in generic formula '" + arg.GetName() + "'"); + if (!dynamic_cast(arg.getParameter(axis.name.c_str()))) { + RooJSONFactoryWSTool::error("axis '" + axis.name + "' is not a real-valued formula variable of " + context); } - - auto *observable = dynamic_cast(arg.getParameter(axisName.c_str())); - if (!observable) { - RooJSONFactoryWSTool::error( - "axis '" + axisName + "' is not a real-valued formula variable of generic formula '" + arg.GetName() + "'"); + auto const *regular = std::get_if(&axis.binning); + if (regular && (regular->nbins < 1 || !(axis.min < axis.max))) { + RooJSONFactoryWSTool::error("axis '" + axis.name + "' of " + context + + " needs a positive \"nbins\" and \"min\" smaller than \"max\""); } - - std::unique_ptr binning = readFormulaAxisBinning(axis, axisName, arg.GetName()); - arg.setBinning(*observable, *binning, /*checkFlatness=*/false); + } + // The formula finds its variable by name, so the variables that are created for the axes can stand in for them. + RooArgSet vars = HS3::createObservables(axes); + for (auto *var : static_range_cast(vars)) { + arg.setBinning(*var, var->getBinning(), /*checkFlatness=*/false); } } template -bool importFormulaArg(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importFormulaArg(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); if (!p.has_child("expression")) { @@ -400,32 +312,43 @@ bool importFormulaArg(RooJSONFactoryWSTool *tool, const JSONNode &p) translateImportedExpression(formula); RooArgList dependents; for (const auto &d : extractArguments(formula.Data())) { - dependents.add(*tool->request(d, name)); + RooAbsReal *dependent = tool->request(d, name); + // HS3 0.3 (HS3 PR #154) only allows expressions to refer to parameters, variables and functions. + if (dynamic_cast(dependent) && !(tool->hs3Version() < HS3::Version{0, 3})) { + RooJSONFactoryWSTool::error("the expression of '" + name + "' refers to the distribution '" + d + + "', which is not allowed in HS3"); + } + dependents.add(*dependent); } RooArg_t arg{name.c_str(), formula, dependents}; importFormulaBinnings(arg, p); tool->wsImport(arg); - return true; } -bool importProduct(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importProduct(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); tool->wsEmplace(name, tool->requestArgList(p, "factors")); - return true; } -bool importProdPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - tool->wsEmplace(name, tool->requestArgList(p, "factors")); - return true; -} +// RooLognormal has two parametrizations. In the standard one, its parameters are the mean "mu" and the width "sigma" of +// log(x), like in the lognormal_dist of HS3. In the other one, which is the default of RooFit, they are the median m0 +// of x and the shape parameter k, with mu = log(m0) and sigma = log(k). Such a distribution is written with "mu" and +// "sigma" naming the functions log(m0) and log(k), which are named after m0 and k with the suffix below. The suffix +// is also how the parametrization is recognized when reading: the distribution is then created from m0 and k +// themselves instead of the functions. Only lognormalLogName() and importLogNormal() know this convention, for +// distributions on their own and for the components of array distributions alike. +constexpr const char *lognormalLogSuffix = "_lognormal_log"; -// The name of the pdf that represents the component of an array distribution over the observable `observable`. -std::string arrayComponentName(std::string const &observable) +// What a RooLognormal that is not in the standard parametrization writes as "mu" (for its median) or "sigma" (for its +// shape parameter): the name of the function log(`param`). The function is written as well if there is a `tool`; +// without one, only the name is given. +std::string lognormalLogName(RooAbsReal const ¶m, RooJSONFactoryWSTool *tool) { - return observable + "_constraint"; + if (tool) { + return tool->exportTransformed(¶m, lognormalLogSuffix, "log(%s)"); + } + return transformedBaseName(param) + lognormalLogSuffix; } // Copy the value of a JSON node, keeping numbers and booleans as such. @@ -449,7 +372,7 @@ void copyValue(JSONNode &target, JSONNode const &source) // name of the array holds them together, so that the array can be written back as `array_dist`. HistFactory modifiers // that name an array as the constraint of a parameter pick their component out of it (see // JSONFactories_HistFactory.cxx). -bool importArrayDist(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importArrayDist(RooJSONFactoryWSTool *tool, const JSONNode &p) { const std::string name = RooJSONFactoryWSTool::name(p); auto error = [&name](std::string const &message) { @@ -508,10 +431,23 @@ bool importArrayDist(RooJSONFactoryWSTool *tool, const JSONNode &p) } observables.push_back(x); - const std::string pdfName = arrayComponentName(x); + const std::string pdfName = x + "_constraint"; + const std::string form = RooFit::HS3::arrayComponentForm(componentType, component, defaults); if (ws.arg(pdfName)) { - error("cannot create the component for '" + x + "' as '" + pdfName + - "', the workspace already has an object of that name"); + // HS3 composes array distributions by union: an identical component of another array distribution of this + // document is the same factor, so the object is shared. A component over the same observable that is not + // identical is an error: the two would be different factors that share a variate. + std::string const *existing = RooFit::HS3::importedArrayComponentForm(*tool, pdfName); + if (!existing) { + error("cannot create the component for '" + x + "' as '" + pdfName + + "', the workspace already has an object of that name"); + } + if (*existing != form) { + error(context + " over '" + x + + "' is not identical to the component over the same observable of another array distribution"); + } + pdfs.add(*ws.pdf(pdfName)); + continue; } JSONNode &node = nodes.append_child().set_map(); node["name"] << pdfName; @@ -529,6 +465,7 @@ bool importArrayDist(RooJSONFactoryWSTool *tool, const JSONNode &p) if (!pdf) { error(context + " did not produce a distribution"); } + RooFit::HS3::recordImportedArrayComponent(*tool, pdfName, form); pdfs.add(*pdf); } @@ -555,127 +492,33 @@ bool importArrayDist(RooJSONFactoryWSTool *tool, const JSONNode &p) RooProdPdf array{name.c_str(), name.c_str(), pdfs}; array.setAttribute(arrayDistTag); + // The product only exists to hold the components together. + array.setAttribute(helperObjectTag); tool->wsImport(array); - return true; -} - -// One field of a component of an array distribution: a name or a number. -using ArrayFieldValue = std::variant; - -struct ArrayComponent { - std::string type; - std::string x; - std::vector> fields; -}; - -ArrayFieldValue nameOrNumber(RooAbsReal const &arg) -{ - if (isLiteralConstVar(arg)) { - return static_cast(arg).getVal(); - } - return std::string{arg.GetName()}; -} - -// The type, observable and fields of `pdf` as a component of an array distribution, with the fields of the standalone -// exporters, or std::nullopt if `pdf` cannot be a component. A Gaussian whose mean is the recorded `observable` is -// written with x and mean swapped. -std::optional arrayComponentOf(RooAbsPdf const &pdf, std::string const *observable = nullptr) -{ - ArrayComponent out; - RooAbsReal const *x = nullptr; - if (auto const *poisson = dynamic_cast(&pdf)) { - out.type = poisson->getNoRounding() ? "poisson_continuous_dist" : "poisson_dist"; - x = &poisson->getX(); - out.fields = {{"mean", nameOrNumber(poisson->getMean())}}; - } else if (auto const *gauss = dynamic_cast(&pdf)) { - out.type = "gaussian_dist"; - x = &gauss->getX(); - RooAbsReal const *mean = &gauss->getMean(); - // A Gaussian constraint built by HistFactory has the parameter as x and the global observable as mean. The - // Gaussian is symmetric in the two, so it is written with the observable as x where the observable is known. - if (observable && *observable == mean->GetName() && !isLiteralConstVar(*mean)) { - std::swap(x, mean); - } - out.fields = {{"mean", nameOrNumber(*mean)}, {"sigma", nameOrNumber(gauss->getSigma())}}; - } else if (auto const *lognormal = dynamic_cast(&pdf)) { - if (!lognormal->useStandardParametrization()) { - return std::nullopt; - } - out.type = "lognormal_dist"; - x = &lognormal->getX(); - out.fields = {{"mu", nameOrNumber(lognormal->getMedian())}, {"sigma", nameOrNumber(lognormal->getShapeK())}}; - } else { - return std::nullopt; - } - // The observable of a component must be named: the observables are pairwise distinct and are paired with data. - if (isLiteralConstVar(*x)) { - return std::nullopt; - } - out.x = x->GetName(); - return out; } -bool importAddition(RooJSONFactoryWSTool *tool, const JSONNode &p) +// A "combined_dist" is interpreted by the document. This reads one that doesn't come with a document. +void importCombinedDist(RooJSONFactoryWSTool *tool, const JSONNode &p) { - std::string name(RooJSONFactoryWSTool::name(p)); - tool->wsEmplace(name, tool->requestArgList(p, "summands")); - return true; -} - -// A "combined_dist" is a RooSimultaneous. Its index category is the single column, or a RooSuperCategory over the -// columns in the order of the document; the columns are categories declared by categorical axes of the domains. -bool importCombinedDist(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string const name(RooJSONFactoryWSTool::name(p)); - std::string const context = "the combined_dist '" + name + "'"; - // Whether a combined distribution is extended follows from its components; HS3 forbids declaring it. - if (p.has_child("extended")) { - RooJSONFactoryWSTool::error(context + " must not have \"extended\": whether it is extended follows from its " + - "components"); - } - std::vector columns = readSelectionColumns(*tool->workspace(), p, "columns", context); - - auto const *componentsNode = p.find("components"); - if (!componentsNode || !componentsNode->is_seq() || componentsNode->num_children() == 0) { - RooJSONFactoryWSTool::error("\"components\" of " + context + " must be a non-empty list"); + HS3::Diagnostics diagnostics; + HS3::Distribution const dist = HS3::parseDistribution(p, diagnostics); + if (!HS3::hasErrors(diagnostics)) { + HS3::validateCombinedDist(std::get(dist.body), dist.name, diagnostics); } - std::map pdfMap; - for (JSONNode const &component : componentsNode->children()) { - if (!component.is_map() || !component.has_child("dist")) { - RooJSONFactoryWSTool::error("every component of " + context + " must have a \"key\" and a \"dist\""); - } - std::vector const key = readSelectionKey(component, columns, context); - std::string const label = selectionLabel(key); - if (pdfMap.find(label) != pdfMap.end()) { - RooJSONFactoryWSTool::error("the key " + selectionKeyToString(key) + " is used by two components of " + - context); - } - pdfMap[label] = tool->request(component["dist"].val(), name); - } - - if (columns.size() == 1) { - tool->wsImport(RooSimultaneous{name.c_str(), name.c_str(), pdfMap, *columns.front()}); - return true; - } - RooArgSet inputs; - for (RooCategory *column : columns) { - inputs.add(*column); - } - std::string const indexName = name + "_index"; - RooSuperCategory indexCat{indexName.c_str(), indexName.c_str(), inputs}; - tool->wsImport(RooSimultaneous{name.c_str(), name.c_str(), pdfMap, indexCat}); - return true; + HS3::report(diagnostics, dist.name); + HS3::buildCombinedDist(*tool, dist.name, std::get(dist.body)); } -bool importAddPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importAddPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); + const std::size_t nSummands = p.has_child("summands") ? p["summands"].num_children() : 0; + const std::size_t nCoefficients = p.has_child("coefficients") ? p["coefficients"].num_children() : 0; + const bool sinceHS3v03 = !(tool->hs3Version() < HS3::Version{0, 3}); if (p.has_child("extended")) { // Check the explicit "extended" flag against the number of coefficients, which determines the extended mode of // the RooAddPdf. const bool extended = p["extended"].val_bool(); - const std::size_t nSummands = p.has_child("summands") ? p["summands"].num_children() : 0; - const std::size_t nCoefficients = p.has_child("coefficients") ? p["coefficients"].num_children() : 0; const std::string counts = std::to_string(nCoefficients) + " coefficients for " + std::to_string(nSummands) + " summands"; if (extended && nCoefficients > 0 && nCoefficients + 1 == nSummands) { @@ -686,31 +529,52 @@ bool importAddPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) RooJSONFactoryWSTool::error("mixture_dist '" + name + "' is declared as not extended, but it has " + counts + ", which defines an extended mixture of extended distributions"); } - if (!extended && nCoefficients == nSummands) { - RooJSONFactoryWSTool::warning("RooFitHS3: mixture_dist '" + name + "' is declared as not extended, but " + - "RooFit can use it as extended because it has one coefficient per summand. " + - "Don't use the extended likelihood term when fitting it."); + if (!extended && nCoefficients == nSummands && nSummands > 0) { + // A RooAddPdf with one coefficient per summand can always be extended. The non-extended mixture is the one + // with the normalized coefficients c_i / sum(c), of which the last one follows from the others. + RooArgList summands = tool->requestArgList(p, "summands"); + RooArgList coefficients = tool->requestArgList(p, "coefficients"); + std::string sum; + for (std::size_t i = 0; i < nCoefficients; ++i) { + sum += (i == 0 ? "x[" : "+x[") + std::to_string(i) + "]"; + } + RooArgList fractions; + for (std::size_t i = 0; i + 1 < nCoefficients; ++i) { + fractions.add(tool->wsEmplace(name + "_fraction_" + std::to_string(i), + ("x[" + std::to_string(i) + "]/(" + sum + ")").c_str(), + coefficients)); + } + tool->wsEmplace(name, summands, fractions); + return; } + } else if (nCoefficients == nSummands && nSummands > 0 && sinceHS3v03) { + // HS3 0.3 requires the flag. Where it decides about the Poisson term, a document without it is invalid. + RooJSONFactoryWSTool::error("mixture_dist '" + name + + "' has one coefficient per summand, but no \"extended\" flag, which HS3 " + "requires to decide whether the mixture is extended"); + } + if (nSummands > 0 && nCoefficients == 0 && sinceHS3v03) { + RooJSONFactoryWSTool::warning("RooFitHS3: mixture_dist '" + name + + "' has no coefficients, which HS3 0.3 doesn't define. It is read as the sum of " + "its extended summands."); } if (!tool->requestArgList(p, "coefficients").empty()) { tool->wsEmplace(name, tool->requestArgList(p, "summands"), tool->requestArgList(p, "coefficients")); - return true; + } else { + tool->wsEmplace(name, tool->requestArgList(p, "summands")); } - tool->wsEmplace(name, tool->requestArgList(p, "summands")); - return true; } -bool importAddModel(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importAddModel(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); tool->wsEmplace(name, tool->requestArgList(p, "summands"), tool->requestArgList(p, "coefficients")); - return true; } template -bool importBinWidthFunction(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importBinWidthFunction(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooHistFunc *hf = dynamic_cast(tool->request(p["histogram"].val(), name)); @@ -718,10 +582,9 @@ bool importBinWidthFunction(RooJSONFactoryWSTool *tool, const JSONNode &p) RooJSONFactoryWSTool::error("histogram '" + p["histogram"].val() + "' of '" + name + "' is not a RooHistFunc"); } tool->wsEmplace(name, *hf, DivideByBinWidth); - return true; } -bool importBinSamplingPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importBinSamplingPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); @@ -740,27 +603,19 @@ bool importBinSamplingPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) double epsilon(p["epsilon"].val_double()); tool->wsEmplace(name, *obs, *pdf, epsilon); - - return true; } -template -bool importRealSum(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importRealSumPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooArgList samples = tool->requestArgList(p, "samples"); RooArgList coefs = tool->requestArgList(p, "coefficients"); - if constexpr (std::is_same_v) { - const bool extended = p.has_child("extended") && p["extended"].val_bool(); - tool->wsEmplace(name, samples, coefs, extended); - } else { - tool->wsEmplace(name, samples, coefs); - } - return true; + const bool extended = p.has_child("extended") && p["extended"].val_bool(); + tool->wsEmplace(name, samples, coefs, extended); } template -bool importPolynomial(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importPolynomial(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); if (!p.has_child("coefficients")) { @@ -785,7 +640,6 @@ bool importPolynomial(RooJSONFactoryWSTool *tool, const JSONNode &p) } tool->wsEmplace(name, *x, coefs, lowestOrder); - return true; } // HS3 0.3 distinguishes the discrete "poisson_dist" from the "poisson_continuous_dist" (HS3 #152), which are a @@ -793,13 +647,13 @@ bool importPolynomial(RooJSONFactoryWSTool *tool, const JSONNode &p) // false denotes the continuous form. That key is read for documents before HS3 0.3, without it the discrete form is // meant. From HS3 0.3 on, it must not be present. template -bool importPoisson(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importPoisson(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); bool noRounding = Continuous; if (p.has_child("integer")) { std::string const type = Continuous ? "poisson_continuous_dist" : "poisson_dist"; - if (Continuous || !isHS3VersionBefore(tool->hs3VersionInput(), 0, 3)) { + if (Continuous || !(tool->hs3Version() < HS3::Version{0, 3})) { RooJSONFactoryWSTool::error( type + " '" + name + "' has the key \"integer\", which is not part of HS3. The discrete Poisson distribution is " @@ -810,10 +664,9 @@ bool importPoisson(RooJSONFactoryWSTool *tool, const JSONNode &p) RooAbsReal *x = tool->requestArg(p, "x"); RooAbsReal *mean = tool->requestArg(p, "mean"); tool->wsEmplace(name, *x, *mean, noRounding); - return true; } -bool importDecay(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importDecay(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooRealVar *t = tool->requestArg(p, "t"); @@ -825,28 +678,9 @@ bool importDecay(RooJSONFactoryWSTool *tool, const JSONNode &p) } RooDecay::DecayType decayType = static_cast(p["decayType"].val_int()); tool->wsEmplace(name, *t, *tau, *model, decayType); - return true; -} - -bool importTruthModel(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooRealVar *x = tool->requestArg(p, "x"); - tool->wsEmplace(name, *x); - return true; -} - -bool importGaussModel(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooRealVar *x = tool->requestArg(p, "x"); - RooRealVar *mean = tool->requestArg(p, "mean"); - RooRealVar *sigma = tool->requestArg(p, "sigma"); - tool->wsEmplace(name, *x, *mean, *sigma); - return true; } -bool importRealIntegral(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importRealIntegral(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooAbsReal *func = tool->requestArg(p, "integrand"); @@ -857,35 +691,27 @@ bool importRealIntegral(RooJSONFactoryWSTool *tool, const JSONNode &p) normSet.add(tool->requestArgSet(p, "normalization")); normSetPtr = &normSet; } - std::string domain; - bool hasDomain = p.has_child("domain"); - if (hasDomain) { - domain = p["domain"].val(); - } + JSONNode const *domain = p.find("domain"); // todo: at some point, take care of integrator configurations tool->wsEmplace(name, *func, vars, normSetPtr, static_cast(nullptr), - hasDomain ? domain.c_str() : nullptr); - return true; + domain ? domain->val().c_str() : nullptr); } -bool importDerivative(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importDerivative(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooAbsReal *func = tool->requestArg(p, "function"); RooRealVar *x = tool->requestArg(p, "x"); Int_t order = p["order"].val_int(); double eps = p["eps"].val_double(); + RooArgSet normSet; if (p.has_child("normalization")) { - RooArgSet normSet; normSet.add(tool->requestArgSet(p, "normalization")); - tool->wsEmplace(name, *func, *x, normSet, order, eps); - return true; } - tool->wsEmplace(name, *func, *x, order, eps); - return true; + tool->wsEmplace(name, *func, *x, normSet, order, eps); } -bool importFFTConvPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importFFTConvPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooRealVar *convVar = tool->requestArg(p, "conv_var"); @@ -895,47 +721,55 @@ bool importFFTConvPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) if (p.has_child("conv_func")) { RooAbsReal *convFunc = tool->requestArg(p, "conv_func"); tool->wsEmplace(name, *convFunc, *convVar, *pdf1, *pdf2, order); - return true; + } else { + tool->wsEmplace(name, *convVar, *pdf1, *pdf2, order); } - tool->wsEmplace(name, *convVar, *pdf1, *pdf2, order); - return true; } -bool importExtendPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importExtendPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); // HS3 calls the fields "dist" and "rate". RooFit used to write "pdf" and "norm", which are still accepted. RooAbsPdf *pdf = tool->requestArg(p, p.has_child("pdf") && !p.has_child("dist") ? "pdf" : "dist"); RooAbsReal *norm = tool->requestArg(p, p.has_child("norm") && !p.has_child("rate") ? "norm" : "rate"); - if (p.has_child("range")) { - std::string rangeName = p["range"].val(); - tool->wsEmplace(name, *pdf, *norm, rangeName.c_str()); - return true; + JSONNode const *range = p.find("range"); + tool->wsEmplace(name, *pdf, *norm, range ? range->val().c_str() : nullptr); +} + +// HS3 0.3 calls the location parameter of the Landau distribution "mu", since it is neither the mean nor the most +// probable value. Documents before 0.3 used "mean", which is still read. +void importLandau(RooJSONFactoryWSTool *tool, const JSONNode &p) +{ + const std::string name(RooJSONFactoryWSTool::name(p)); + const bool legacyKey = !p.has_child("mu") && p.has_child("mean"); + if (legacyKey && !(tool->hs3Version() < HS3::Version{0, 3})) { + RooJSONFactoryWSTool::warning("RooFitHS3: the landau_dist '" + name + + "' uses the key \"mean\", which HS3 0.3 renamed to \"mu\"."); } - tool->wsEmplace(name, *pdf, *norm); - return true; + RooAbsReal *x = tool->requestArg(p, "x"); + RooAbsReal *mu = tool->requestArg(p, legacyKey ? "mean" : "mu"); + RooAbsReal *sigma = tool->requestArg(p, "sigma"); + tool->wsEmplace(name, *x, *mu, *sigma); } -bool importLogNormal(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importLogNormal(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooAbsReal *x = tool->requestArg(p, "x"); // Same mechanism to undo the parameter transformation as in the // importExponential() function (see comments in that function for more info). - const std::string muName = p["mu"].val(); - const std::string sigmaName = p["sigma"].val(); - const bool isTransformed = endsWith(muName, "_lognormal_log"); - const std::string suffixToRemove = isTransformed ? "_lognormal_log" : ""; - RooAbsReal *mu = tool->request(removeSuffix(muName, suffixToRemove), name); - RooAbsReal *sigma = tool->request(removeSuffix(sigmaName, suffixToRemove), name); - - tool->wsEmplace(name, *x, *mu, *sigma, !isTransformed); + const std::string mu = p["mu"].val(); + const std::string sigma = p["sigma"].val(); + const bool isStandard = !endsWith(mu, lognormalLogSuffix); + RooAbsReal *median = tool->request(isStandard ? mu : transformedOriginal(mu, lognormalLogSuffix), name); + RooAbsReal *shape = + tool->request(isStandard ? sigma : transformedOriginal(sigma, lognormalLogSuffix), name); - return true; + tool->wsEmplace(name, *x, *median, *shape, isStandard); } -bool importExponential(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importExponential(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooAbsReal *x = tool->requestArg(p, "x"); @@ -973,69 +807,49 @@ bool importExponential(RooJSONFactoryWSTool *tool, const JSONNode &p) const std::string constParamName = p["c"].val(); const bool isInverted = endsWith(constParamName, "_exponential_inverted"); - const std::string suffixToRemove = isInverted ? "_exponential_inverted" : ""; - RooAbsReal *c = tool->request(removeSuffix(constParamName, suffixToRemove), name); + RooAbsReal *c = tool->request( + isInverted ? transformedOriginal(constParamName, "_exponential_inverted") : constParamName, name); tool->wsEmplace(name, *x, *c, !isInverted); - - return true; } -bool importMultiVarGaussian(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importMultiVarGaussian(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); - bool has_cov = p.has_child("covariances"); - bool has_corr = p.has_child("correlations") && p.has_child("standard_deviations"); - if (!has_cov && !has_corr) { + const bool hasCov = p.has_child("covariances"); + const bool hasCorr = p.has_child("correlations") && p.has_child("standard_deviations"); + if (!hasCov && !hasCorr) { RooJSONFactoryWSTool::error("no covariances or correlations+standard_deviations given in '" + name + "'"); } - - TMatrixDSym covmat; - + // The covariances are given directly, or as correlations that are scaled with the standard deviations. + const std::string key = hasCov ? "covariances" : "correlations"; + std::vector sigmas; + if (hasCov) { + sigmas.assign(p[key].num_children(), 1.0); + } else { + sigmas << p["standard_deviations"]; + } + const std::size_t n = sigmas.size(); // The matrix must be square, otherwise the entries would be written out of bounds. - auto checkSquare = [&](const char *key, std::size_t n) { - if (p[key].num_children() != n) { - RooJSONFactoryWSTool::error("\"" + std::string(key) + "\" in '" + name + "' must have " + std::to_string(n) + - " rows"); - } - for (const auto &row : p[key].children()) { - if (!row.is_seq() || row.num_children() != n) { - RooJSONFactoryWSTool::error("\"" + std::string(key) + "\" in '" + name + "' must be a square matrix"); - } - } - }; - - if (has_cov) { - int n = p["covariances"].num_children(); - checkSquare("covariances", n); - int i = 0; - covmat.ResizeTo(n, n); - for (const auto &row : p["covariances"].children()) { - int j = 0; - for (const auto &val : row.children()) { - covmat(i, j) = val.val_double(); - ++j; - } - ++i; + if (p[key].num_children() != n) { + RooJSONFactoryWSTool::error("\"" + key + "\" in '" + name + "' must have " + std::to_string(n) + " rows"); + } + TMatrixDSym covmat; + covmat.ResizeTo(n, n); + int i = 0; + for (const auto &row : p[key].children()) { + if (!row.is_seq() || row.num_children() != n) { + RooJSONFactoryWSTool::error("\"" + key + "\" in '" + name + "' must be a square matrix"); } - } else { - std::vector variances; - variances << p["standard_deviations"]; - checkSquare("correlations", variances.size()); - covmat.ResizeTo(variances.size(), variances.size()); - int i = 0; - for (const auto &row : p["correlations"].children()) { - int j = 0; - for (const auto &val : row.children()) { - covmat(i, j) = val.val_double() * variances[i] * variances[j]; - ++j; - } - ++i; + int j = 0; + for (const auto &val : row.children()) { + covmat(i, j) = val.val_double() * sigmas[i] * sigmas[j]; + ++j; } + ++i; } tool->wsEmplace(name, tool->requestArgList(p, "x"), tool->requestArgList(p, "mean"), covmat); - return true; } /// Read the binning variables from the "axes" node, ordered like in `varList`. @@ -1058,54 +872,38 @@ bool importParamHistFunc(RooJSONFactoryWSTool *tool, const JSONNode &p) } std::string name(RooJSONFactoryWSTool::name(p)); RooArgList varList = tool->requestArgList(p, "variables"); - if (!p.has_child("axes")) { - std::stringstream ss; - ss << "No axes given in '" << name << "'" - << ". Using default binning (uniform; nbins=100). If needed, export the Workspace to JSON with a newer " - << "Root version that supports custom ParamHistFunc binnings(>=6.38.00)." << std::endl; - RooJSONFactoryWSTool::warning(ss.str()); - checkParamHistFuncOrderIsDefined(tool->hs3VersionInput(), varList, "the ParamHistFunc '" + name + "'"); - tool->wsEmplace(name, varList, - toParamHistFuncOrder(tool->requestArgList(p, "parameters"), varList)); - return true; - } - RooArgList binning = readBinning(p, varList); - checkParamHistFuncOrderIsDefined(tool->hs3VersionInput(), binning, "the ParamHistFunc '" + name + "'"); + const bool hasAxes = p.has_child("axes"); + if (!hasAxes) { + RooJSONFactoryWSTool::warning( + "No axes given in '" + name + + "'. Using default binning (uniform; nbins=100). If needed, export the Workspace to JSON with a newer " + "Root version that supports custom ParamHistFunc binnings(>=6.38.00).\n"); + } + RooArgList binning = hasAxes ? readBinning(p, varList) : varList; + checkParamHistFuncOrderIsDefined(tool->hs3Version(), binning, "the ParamHistFunc '" + name + "'"); // The parameters are given in the order of HS3 (last axis fastest); ParamHistFunc expects TH1 order. tool->wsEmplace(name, binning, toParamHistFuncOrder(tool->requestArgList(p, "parameters"), binning)); return true; } -bool importSpline(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importSpline(RooJSONFactoryWSTool *tool, const JSONNode &p) { const std::string name(RooJSONFactoryWSTool::name(p)); - - // Mandatory fields - if (!p.has_child("x")) { - RooJSONFactoryWSTool::error("no x given in '" + name + "'"); - } + RooAbsReal *x = tool->requestArg(p, "x"); if (!p.has_child("x0") || !p.has_child("y0")) { RooJSONFactoryWSTool::error("no x0/y0 given in '" + name + "'"); } - RooAbsReal *x = tool->requestArg(p, "x"); - - // Optional fields (defaults follow RooSpline ctor defaults) - std::string algo = p.has_child("interpolation") ? p["interpolation"].val() : "poly3"; - int order = 0; - if (algo == "poly3") { - order = 3; - } else if (algo == "poly5") { - order = 5; - } else { + // The optional fields have the defaults of RooSpline. + const std::string algo = p.has_child("interpolation") ? p["interpolation"].val() : "poly3"; + if (algo != "poly3" && algo != "poly5") { RooJSONFactoryWSTool::error("unsupported algo '" + algo + "' for RooSpline in '" + name + "': allowed are 'poly3' and 'poly5'"); } - const bool logx = p.has_child("logx") ? p["logx"].val_bool() : false; - const bool logy = p.has_child("logy") ? p["logy"].val_bool() : false; + const bool logx = p.has_child("logx") && p["logx"].val_bool(); + const bool logy = p.has_child("logy") && p["logy"].val_bool(); - // Read knots std::vector x0; std::vector y0; x0 << p["x0"]; @@ -1119,25 +917,10 @@ bool importSpline(RooJSONFactoryWSTool *tool, const JSONNode &p) RooJSONFactoryWSTool::error("need at least 2 knots in '" + name + "'"); } - // Construct RooSpline(name,title, x, x0, y0, order, logx, logy) - tool->wsEmplace<::RooSpline>(name.c_str(), *x, std::span(x0.data(), x0.size()), - std::span(y0.data(), y0.size()), order, logx, logy); - - return true; + tool->wsEmplace<::RooSpline>(name.c_str(), *x, x0, y0, algo == "poly5" ? 5 : 3, logx, logy); } -bool importArgus(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *mass = tool->requestArg(p, "mass"); - RooAbsReal *resonance = tool->requestArg(p, "resonance"); - RooAbsReal *slope = tool->requestArg(p, "slope"); - RooAbsReal *power = tool->requestArg(p, "power"); - tool->wsEmplace(name, *mass, *resonance, *slope, *power); - return true; -} - -bool importBernstein(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importBernstein(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); auto *x = dynamic_cast(tool->requestArg(p, "x")); @@ -1145,122 +928,28 @@ bool importBernstein(RooJSONFactoryWSTool *tool, const JSONNode &p) RooJSONFactoryWSTool::error("\"x\" of '" + name + "' is not a variable"); } tool->wsEmplace(name, *x, tool->requestArgList(p, "coefficients")); - return true; -} - -bool importBifurGauss(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *x = tool->requestArg(p, "x"); - RooAbsReal *mean = tool->requestArg(p, "mean"); - RooAbsReal *sigmaL = tool->requestArg(p, "sigmaL"); - RooAbsReal *sigmaR = tool->requestArg(p, "sigmaR"); - tool->wsEmplace(name, *x, *mean, *sigmaL, *sigmaR); - return true; -} - -bool importChebychev(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *x = tool->requestArg(p, "x"); - tool->wsEmplace(name, *x, tool->requestArgList(p, "coefficients")); - return true; -} - -bool importEffProd(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsPdf *pdf = tool->requestArg(p, "pdf"); - RooAbsReal *eff = tool->requestArg(p, "eff"); - tool->wsEmplace(name, *pdf, *eff); - return true; -} - -bool importGamma(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *x = tool->requestArg(p, "x"); - RooAbsReal *gamma = tool->requestArg(p, "gamma"); - RooAbsReal *beta = tool->requestArg(p, "beta"); - RooAbsReal *mu = tool->requestArg(p, "mu"); - tool->wsEmplace(name, *x, *gamma, *beta, *mu); - return true; } -bool importGaussian(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importGaussian(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); RooAbsReal *x = tool->requestArg(p, "x"); RooAbsReal *mean = tool->requestArg(p, "mean"); RooAbsReal *sigma = tool->requestArg(p, "sigma"); tool->wsEmplace(name, *x, *mean, *sigma); - return true; } -bool importLandau(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *x = tool->requestArg(p, "x"); - RooAbsReal *mean = tool->requestArg(p, "mean"); - RooAbsReal *sigma = tool->requestArg(p, "sigma"); - tool->wsEmplace(name, *x, *mean, *sigma); - return true; -} - -bool importPowerSum(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *x = tool->requestArg(p, "x"); - RooArgList coefs = tool->requestArgList(p, "coefficients"); - RooArgList exps = tool->requestArgList(p, "exponents"); - tool->wsEmplace(name, *x, coefs, exps); - return true; -} - -bool importProjectedPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) -{ - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsPdf *pdf = tool->requestArg(p, "input_pdf"); - tool->wsEmplace(name, *pdf, tool->requestArgSet(p, "observables")); - return true; -} - -bool importUniform(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importUniform(RooJSONFactoryWSTool *tool, const JSONNode &p) { std::string name(RooJSONFactoryWSTool::name(p)); // A single observable may also be given by its name. if (p.has_child("x") && !p["x"].is_seq()) { tool->wsEmplace(name, RooArgSet{*tool->requestArg(p, "x")}); - return true; + } else { + tool->wsEmplace(name, tool->requestArgSet(p, "x")); } - tool->wsEmplace(name, tool->requestArgSet(p, "x")); - return true; } -// Importer that is registered with top priority for a deprecated type key. It warns once, and then lets the importers -// that are registered for the deprecated key do the actual work. -class DeprecatedKeyImporter : public RooFit::JSONIO::Importer { -public: - DeprecatedKeyImporter(std::string key, std::string replacement) - : _key{std::move(key)}, _replacement{std::move(replacement)} - { - } - bool importArg(RooJSONFactoryWSTool *, const JSONNode &) const override - { - if (!_warned) { - RooJSONFactoryWSTool::warning("RooFitHS3: the type \"" + _key + "\" is deprecated, please use \"" + - _replacement + "\" instead."); - _warned = true; - } - return false; - } - -private: - const std::string _key; - const std::string _replacement; - mutable bool _warned = false; -}; - namespace CrystalBall { // The keys of the tails of the double-sided Crystal Ball. @@ -1268,12 +957,7 @@ const std::initializer_list tailKeys = {"alpha_L", "n_L", "alpha_R bool hasAny(const JSONNode &p, std::initializer_list keys) { - for (const char *key : keys) { - if (p.has_child(key)) { - return true; - } - } - return false; + return std::any_of(keys.begin(), keys.end(), [&p](const char *key) { return p.has_child(key); }); } bool hasDoubleSidedKeys(const JSONNode &p) @@ -1334,39 +1018,27 @@ void importDoubleSided(RooJSONFactoryWSTool *tool, const JSONNode &p, std::strin } // namespace CrystalBall // "crystalball_dist" is the single-sided Crystal Ball (RooCBShape). Before HS3 0.3, it was also the double-sided -// Crystal Ball, recognizable by the "_L"/"_R" fields. Such documents are still read, with a deprecation warning. -class CrystalBallImporter : public RooFit::JSONIO::Importer { -public: - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override - { - if (!CrystalBall::hasDoubleSidedKeys(p)) { - std::string name(RooJSONFactoryWSTool::name(p)); - RooAbsReal *m = tool->requestArg(p, "m"); - RooAbsReal *m0 = tool->requestArg(p, "m0"); - RooAbsReal *sigma = tool->requestArg(p, "sigma"); - RooAbsReal *alpha = tool->requestArg(p, "alpha"); - RooAbsReal *n = tool->requestArg(p, "n"); - tool->wsEmplace(name, *m, *m0, *sigma, *alpha, *n); - return true; - } - if (!_warned) { - RooJSONFactoryWSTool::warning("RooFitHS3: \"crystalball_dist\" with the fields \"alpha_L\", \"n_L\", " - "\"alpha_R\", \"n_R\", \"sigma_L\" or \"sigma_R\" is deprecated, please use " - "\"crystalball_doublesided_dist\" for the double-sided Crystal Ball."); - _warned = true; - } - CrystalBall::importDoubleSided(tool, p, "crystalball_dist", /*symmetricShorthand=*/true); - return true; +// Crystal Ball, recognizable by the "_L"/"_R" fields. Such documents are still read by this importer, with a +// deprecation warning. It declines the single-sided Crystal Ball. +bool importLegacyDoubleSidedCrystalBall(RooJSONFactoryWSTool *tool, const JSONNode &p) +{ + if (!CrystalBall::hasDoubleSidedKeys(p)) { + return false; } + static bool warned = false; + if (!warned) { + RooJSONFactoryWSTool::warning("RooFitHS3: \"crystalball_dist\" with the fields \"alpha_L\", \"n_L\", " + "\"alpha_R\", \"n_R\", \"sigma_L\" or \"sigma_R\" is deprecated, please use " + "\"crystalball_doublesided_dist\" for the double-sided Crystal Ball."); + warned = true; + } + CrystalBall::importDoubleSided(tool, p, "crystalball_dist", /*symmetricShorthand=*/true); + return true; +} -private: - mutable bool _warned = false; -}; - -bool importCrystalBallDoubleSided(RooJSONFactoryWSTool *tool, const JSONNode &p) +void importCrystalBallDoubleSided(RooJSONFactoryWSTool *tool, const JSONNode &p) { CrystalBall::importDoubleSided(tool, p, "crystalball_doublesided_dist", /*symmetricShorthand=*/false); - return true; } /////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1376,67 +1048,58 @@ bool importCrystalBallDoubleSided(RooJSONFactoryWSTool *tool, const JSONNode &p) // A RooCrystalBall with one tail is the single-sided "crystalball_dist" (read back as RooCBShape, which follows the // same convention for the sign of alpha). With two tails, it is a "crystalball_doublesided_dist", whose tails are // always written as "alpha_L"/"n_L"/"alpha_R"/"n_R": there is no shorthand for symmetric tails. -bool exportCrystalBall(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportCrystalBall(RooJSONFactoryWSTool *, RooCrystalBall const &pdf, JSONNode &elem) { - auto const *pdf = static_cast(func); - writeReference(elem["m"], pdf->x()); - writeReference(elem["m0"], pdf->x0()); - if (!pdf->alphaR() || !pdf->nR()) { + writeReference(elem["m"], pdf.x()); + writeReference(elem["m0"], pdf.x0()); + if (!pdf.alphaR() || !pdf.nR()) { elem["type"] << "crystalball_dist"; - writeReference(elem["sigma"], pdf->sigmaL()); - writeReference(elem["alpha"], pdf->alphaL()); - writeReference(elem["n"], pdf->nL()); - return true; - } - elem["type"] << key; - if (&pdf->sigmaL() == &pdf->sigmaR()) { - writeReference(elem["sigma"], pdf->sigmaL()); + writeReference(elem["sigma"], pdf.sigmaL()); + writeReference(elem["alpha"], pdf.alphaL()); + writeReference(elem["n"], pdf.nL()); + return; + } + if (&pdf.sigmaL() == &pdf.sigmaR()) { + writeReference(elem["sigma"], pdf.sigmaL()); } else { - writeReference(elem["sigma_L"], pdf->sigmaL()); - writeReference(elem["sigma_R"], pdf->sigmaR()); + writeReference(elem["sigma_L"], pdf.sigmaL()); + writeReference(elem["sigma_R"], pdf.sigmaR()); } - writeReference(elem["alpha_L"], pdf->alphaL()); - writeReference(elem["n_L"], pdf->nL()); - writeReference(elem["alpha_R"], *pdf->alphaR()); - writeReference(elem["n_R"], *pdf->nR()); - return true; + writeReference(elem["alpha_L"], pdf.alphaL()); + writeReference(elem["n_L"], pdf.nL()); + writeReference(elem["alpha_R"], *pdf.alphaR()); + writeReference(elem["n_R"], *pdf.nR()); } template -bool exportAddPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportAddPdf(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { - const RooArg_t *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["summands"], pdf->pdfList()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - elem["extended"] << (pdf->extendMode() != RooArg_t::CanNotBeExtended); - return true; + RooJSONFactoryWSTool::fillSeq(elem["summands"], pdf.pdfList()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); + elem["extended"] << (pdf.extendMode() != RooArg_t::CanNotBeExtended); + // A RooAddModel is written as a mixture_model, which is not part of HS3. + if (pdf.coefList().empty() && std::is_same_v) { + RooJSONFactoryWSTool::warning("RooFitHS3: the RooAddPdf '" + std::string(pdf.GetName()) + + "' has no coefficients. It is written as a mixture_dist without coefficients, " + "which HS3 0.3 doesn't define, so other implementations might not read it."); + } } -template -bool exportRealSum(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportRealSumPdf(RooJSONFactoryWSTool *, RooRealSumPdf const &pdf, JSONNode &elem) { - auto const *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["samples"], pdf->funcList()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - if constexpr (std::is_same_v) { - elem["extended"] << (pdf->extendMode() != RooAbsPdf::CanNotBeExtended); - } - return true; + RooJSONFactoryWSTool::fillSeq(elem["samples"], pdf.funcList()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); + elem["extended"] << (pdf.extendMode() != RooAbsPdf::CanNotBeExtended); } template -bool exportHist(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportHist(RooJSONFactoryWSTool *tool, RooArg_t const &hf, JSONNode &elem) { - const RooArg_t *hf = static_cast(func); - elem["type"] << key; - RooDataHist const &dh = hf->dataHist(); + RooDataHist const &dh = hf.dataHist(); // The axes are named after the observables of the function, which are not necessarily the variables of the // histogram. - auto vars = histogramVariablesNamedAfterObservables(dh, hf->getHistObsList(), hf->variables()); + auto vars = histogramVariablesNamedAfterObservables(dh, hf.getHistObsList(), hf.variables()); tool->exportHisto(*vars, dh.numEntries(), dh.weightArray(), elem["data"].set_map()); - return true; } template @@ -1452,22 +1115,20 @@ bool importHist(RooJSONFactoryWSTool *tool, const JSONNode &p) return true; } -bool exportBinSamplingPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportBinSamplingPdf(RooJSONFactoryWSTool *, RooBinSamplingPdf const &pdf, JSONNode &elem) { - const RooBinSamplingPdf *pdf = static_cast(func); - elem["type"] << key; - elem["pdf"] << pdf->pdf().GetName(); - elem["observable"] << pdf->observable().GetName(); - elem["epsilon"] << pdf->epsilon(); - return true; + elem["pdf"] << pdf.pdf().GetName(); + elem["observable"] << pdf.observable().GetName(); + elem["epsilon"] << pdf.epsilon(); } -bool exportBinWidthFunction(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &) +void exportBinWidthFunction(RooJSONFactoryWSTool *, RooBinWidthFunction const &pdf, JSONNode &elem) { - const RooBinWidthFunction *pdf = static_cast(func); - elem["type"] << (pdf->divideByBinWidth() ? "inverse_binvolume" : "binvolume"); - elem["histogram"] << pdf->histFunc().GetName(); - return true; + // It is registered for "binvolume". + if (pdf.divideByBinWidth()) { + elem["type"] << "inverse_binvolume"; + } + elem["histogram"] << pdf.histFunc().GetName(); } void cleanExpression(TString &expr) @@ -1502,37 +1163,23 @@ void cleanExpression(TString &expr) replaceIdentifier(expr, "TMath::E()", "EULER"); } -bool ternariesToIfElse(std::string const &in, std::string &out); +std::optional ternariesToIfElse(std::string const &in); // Rewrite one comma-free part of an expression, see ternariesToIfElse(). -bool ternaryPartToIfElse(std::string const &in, std::string &out) +std::optional ternaryPartToIfElse(std::string const &in) { - // Find the first '?' that is not enclosed in brackets. A ':' that is not part of "::" can only follow a '?'. - std::size_t question = std::string::npos; - int depth = 0; - for (std::size_t i = 0; i < in.size() && question == std::string::npos; ++i) { - const char c = in[i]; - if (c == '(' || c == '[') { - ++depth; - } else if (c == ')' || c == ']') { - --depth; - } else if (c == ':' && i + 1 < in.size() && in[i + 1] == ':') { - ++i; - } else if (depth == 0 && c == ':') { - return false; - } else if (depth == 0 && c == '?') { - question = i; - } else if (depth == 0 && c == ',') { - return false; - } + // Find the first '?' that is not enclosed in brackets. A ':' can only follow a '?', and a part has no commas. + const std::size_t question = findTopLevel(in, "?:,"); + if (question != std::string::npos && in[question] != '?') { + return std::nullopt; } if (question == std::string::npos) { // No ternary on this level: copy the expression and rewrite the ternaries inside of brackets. - out.clear(); + std::string out; for (std::size_t i = 0; i < in.size(); ++i) { if (in[i] == ')' || in[i] == ']') { - return false; + return std::nullopt; } if (in[i] != '(' && in[i] != '[') { out += in[i]; @@ -1540,91 +1187,79 @@ bool ternaryPartToIfElse(std::string const &in, std::string &out) } const std::size_t close = findClosingBracket(in, i); if (close == std::string::npos) { - return false; + return std::nullopt; } const std::string inner = in.substr(i + 1, close - i - 1); - std::string innerOut; - if (!ternariesToIfElse(inner, innerOut)) { - return false; + const std::optional innerOut = ternariesToIfElse(inner); + if (!innerOut) { + return std::nullopt; } // Parentheses that only group a ternary are redundant around the ifelse() function call. const std::string before = trimmed(in.substr(0, i)); const bool isGrouping = in[i] == '(' && (before.empty() || !isIdentifierChar(before.back())); - if (isGrouping && innerOut != trimmed(inner) && startsWith(innerOut, "ifelse(") && - findClosingBracket(innerOut, std::string("ifelse").size()) == innerOut.size() - 1) { - out += innerOut; + if (isGrouping && *innerOut != trimmed(inner) && startsWith(*innerOut, "ifelse(") && + findClosingBracket(*innerOut, std::string("ifelse").size()) == innerOut->size() - 1) { + out += *innerOut; } else { - out += in[i] + innerOut + in[close]; + out += in[i] + *innerOut + in[close]; } i = close; } - return true; + return out; } - // Find the ':' that belongs to this '?', taking nested ternaries into account. - std::size_t colon = std::string::npos; - int nested = 0; - depth = 0; - for (std::size_t i = question + 1; i < in.size() && colon == std::string::npos; ++i) { - const char c = in[i]; - if (c == '(' || c == '[') { - ++depth; - } else if (c == ')' || c == ']') { - --depth; - } else if (c == ':' && i + 1 < in.size() && in[i + 1] == ':') { - ++i; - } else if (depth == 0 && c == '?') { - ++nested; - } else if (depth == 0 && c == ':') { - if (nested == 0) { - colon = i; - } else { - --nested; - } + // Find the ':' that belongs to this '?': a nested '?' comes with a ':' of its own, which is skipped. + std::size_t colon = question; + for (int nQuestions = 1; nQuestions > 0;) { + colon = findTopLevel(in, "?:", colon + 1); + if (colon == std::string::npos) { + return std::nullopt; } - } - if (colon == std::string::npos) { - return false; + nQuestions += in[colon] == '?' ? 1 : -1; } - std::string condition; - std::string ifTrue; - std::string ifFalse; - if (!ternaryPartToIfElse(stripEnclosingParentheses(in.substr(0, question)), condition) || - !ternaryPartToIfElse(stripEnclosingParentheses(in.substr(question + 1, colon - question - 1)), ifTrue) || - !ternaryPartToIfElse(stripEnclosingParentheses(in.substr(colon + 1)), ifFalse)) { - return false; - } - if (condition.empty() || ifTrue.empty() || ifFalse.empty()) { - return false; + const auto condition = ternaryPartToIfElse(stripEnclosingParentheses(in.substr(0, question))); + const auto ifTrue = ternaryPartToIfElse(stripEnclosingParentheses(in.substr(question + 1, colon - question - 1))); + const auto ifFalse = ternaryPartToIfElse(stripEnclosingParentheses(in.substr(colon + 1))); + if (!condition || !ifTrue || !ifFalse || condition->empty() || ifTrue->empty() || ifFalse->empty()) { + return std::nullopt; } - out = "ifelse(" + condition + "," + ifTrue + "," + ifFalse + ")"; - return true; + return "ifelse(" + *condition + "," + *ifTrue + "," + *ifFalse + ")"; } // Rewrite the C-style ternaries "c ? a : b" in a TFormula expression into the HS3 conditional "ifelse(c,a,b)". Returns -// false if this is not safely possible, e.g. because of unbalanced brackets. -bool ternariesToIfElse(std::string const &in, std::string &out) +// std::nullopt if this is not safely possible, e.g. because of unbalanced brackets. +std::optional ternariesToIfElse(std::string const &in) { - out.clear(); + std::string out; bool first = true; for (std::string const &part : splitTopLevelCommas(in)) { - std::string partOut; - if (!ternaryPartToIfElse(part, partOut)) { - return false; + const std::optional partOut = ternaryPartToIfElse(part); + if (!partOut) { + return std::nullopt; } - out += (first ? "" : ",") + partOut; + out += (first ? "" : ",") + *partOut; first = false; } - return true; + return out; +} + +// HS3 0.3 (HS3 PR #154) only allows expressions to refer to parameters, variables and functions. +void checkExpressionRefersToNoPdf(RooAbsArg const &formula, RooAbsCollection const &dependents) +{ + for (RooAbsArg const *dependent : dependents) { + if (dynamic_cast(dependent)) { + RooJSONFactoryWSTool::error(std::string("the expression of '") + formula.GetName() + "' refers to the pdf '" + + dependent->GetName() + "', which HS3 doesn't allow"); + } + } } template -bool exportFormulaArg(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportFormulaArg(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { - const RooArg_t *pdf = static_cast(func); - elem["type"] << key; - TString expression(pdf->expression()); + checkExpressionRefersToNoPdf(pdf, pdf.dependents()); + TString expression(pdf.expression()); cleanExpression(expression); // If the tokens follow the "x[#]" convention, the square braces enclosing each number // ensures that there is a unique mapping between the token and parameter name @@ -1632,638 +1267,471 @@ bool exportFormulaArg(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &e // So there may be tokens with numbers whose lower place value forms a subset string of ones with a higher place // value, e.g. "@1" is a subset of "@10". So the names of these parameters must be applied descending from the // highest place value in order to ensure each parameter name is uniquely applied to its token. - for (size_t idx = pdf->nParameters(); idx--;) { - const RooAbsArg *par = pdf->getParameter(idx); + for (size_t idx = pdf.nParameters(); idx--;) { + const RooAbsArg *par = pdf.getParameter(idx); expression.ReplaceAll(("x[" + std::to_string(idx) + "]").c_str(), par->GetName()); expression.ReplaceAll(("@" + std::to_string(idx)).c_str(), par->GetName()); } if (expression.Contains("?")) { - std::string rewritten; - if (ternariesToIfElse(expression.Data(), rewritten)) { - expression = rewritten.c_str(); + if (const auto rewritten = ternariesToIfElse(expression.Data())) { + expression = rewritten->c_str(); } else { RooJSONFactoryWSTool::warning(std::string("RooFitHS3: could not translate the ternary operators in the " "expression of '") + - func->GetName() + "' to ifelse(), exporting the expression unchanged."); + pdf.GetName() + "' to ifelse(), exporting the expression unchanged."); } } elem["expression"] << expression.Data(); - for (const RooAbsArg *dependent : pdf->dependents()) { + for (const RooAbsArg *dependent : pdf.dependents()) { auto const *observable = dynamic_cast(dependent); if (!observable) { continue; } - const RooAbsBinning *binning = pdf->getBinning(*observable); + const RooAbsBinning *binning = pdf.getBinning(*observable); if (!binning) { continue; } - auto &axes = elem["axes"]; - if (!axes.is_seq()) { - axes.set_seq(); - } - auto &axis = axes.append_child().set_map(); - axis["name"] << observable->GetName(); - writeAxisBinning(axis, *binning); + HS3::writeAxis(elem["axes"].set_seq().append_child(), HS3::toBinnedAxis(observable->GetName(), *binning)); } - return true; } // Write the "x" reference and the coefficient list for polynomial-like // pdfs/funcs, including the implicit defaults below "lowestOrder" so that the // output is self-documenting. template -bool exportPolynomial(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportPolynomial(RooJSONFactoryWSTool *, RooArg_t const &pdf, JSONNode &elem) { - auto const *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->x().GetName(); + elem["x"] << pdf.x().GetName(); auto &coefs = elem["coefficients"].set_seq(); - for (int i = 0; i < pdf->lowestOrder(); ++i) { + for (int i = 0; i < pdf.lowestOrder(); ++i) { coefs.append_child() << (i == 0 ? 1.0 : 0.0); } - for (const auto &coef : pdf->coefList()) { - coefs.append_child() << coef->GetName(); + for (const auto &coef : pdf.coefList()) { + writeReference(coefs.append_child(), *coef); } - return true; } -bool exportPoisson(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportPoisson(RooJSONFactoryWSTool *, RooPoisson const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << (pdf->getNoRounding() ? "poisson_continuous_dist" : key); - elem["x"] << pdf->getX().GetName(); - elem["mean"] << pdf->getMean().GetName(); - return true; + // It is registered for "poisson_dist", the discrete Poisson distribution. + if (pdf.getNoRounding()) { + elem["type"] << "poisson_continuous_dist"; + } + writeReference(elem["x"], pdf.getX()); + writeReference(elem["mean"], pdf.getMean()); } // The RooDecay servers are the internal resModel-times-basis convolutions, // which are implementation details that should not leak into the JSON. We // only export the actual dependents (t, tau and the original resolution // model) explicitly. -bool exportDecay(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto *pdf = static_cast(func); - elem["type"] << key; - elem["t"] << pdf->getT().GetName(); - elem["tau"] << pdf->getTau().GetName(); - elem["resolutionModel"] << pdf->getModel().GetName(); - elem["decayType"] << pdf->getDecayType(); - - tool->queueExport(pdf->getT()); - tool->queueExport(pdf->getTau()); - tool->queueExport(pdf->getModel()); - - return true; -} - -bool exportTruthModel(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportDecay(RooJSONFactoryWSTool *tool, RooDecay const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->convVar().GetName(); + elem["t"] << pdf.getT().GetName(); + writeReference(elem["tau"], pdf.getTau()); + elem["resolutionModel"] << pdf.getModel().GetName(); + elem["decayType"] << pdf.getDecayType(); - return true; -} - -bool exportGaussModel(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->convVar().GetName(); - elem["mean"] << pdf->getMean().GetName(); - elem["sigma"] << pdf->getSigma().GetName(); - return true; + tool->queueExport(pdf.getT()); + tool->queueExport(pdf.getTau()); + tool->queueExport(pdf.getModel()); } // A Gaussian constraint whose recorded observable (see RooJSONFactoryWSTool::setConstraintObservable()) is its // mean is written with x and mean swapped, so that the observable is x. -bool exportGaussian(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto const *gauss = static_cast(func); - std::string const *observable = tool->constraintObservable(gauss->GetName()); - const bool swap = observable && *observable == gauss->getMean().GetName() && !isLiteralConstVar(gauss->getMean()); - elem["type"] << key; - writeReference(elem["x"], swap ? gauss->getMean() : gauss->getX()); - writeReference(elem["mean"], swap ? gauss->getX() : gauss->getMean()); - writeReference(elem["sigma"], gauss->getSigma()); - return true; +void exportGaussian(RooJSONFactoryWSTool *tool, RooGaussian const &gauss, JSONNode &elem) +{ + std::string const *observable = tool->constraintObservable(gauss.GetName()); + const bool swap = observable && *observable == gauss.getMean().GetName() && !isLiteralConstVar(gauss.getMean()); + writeReference(elem["x"], swap ? gauss.getMean() : gauss.getX()); + writeReference(elem["mean"], swap ? gauss.getX() : gauss.getMean()); + writeReference(elem["sigma"], gauss.getSigma()); } -bool exportLogNormal(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportLogNormal(RooJSONFactoryWSTool *tool, RooLognormal const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); + writeReference(elem["x"], pdf.getX()); - elem["type"] << key; - elem["x"] << pdf->getX().GetName(); + auto &m0 = pdf.getMedian(); + auto &k = pdf.getShapeK(); - auto &m0 = pdf->getMedian(); - auto &k = pdf->getShapeK(); - - if (pdf->useStandardParametrization()) { - elem["mu"] << m0.GetName(); - elem["sigma"] << k.GetName(); + if (pdf.useStandardParametrization()) { + writeReference(elem["mu"], m0); + writeReference(elem["sigma"], k); } else { - elem["mu"] << tool->exportTransformed(&m0, "_lognormal_log", "log(%s)"); - elem["sigma"] << tool->exportTransformed(&k, "_lognormal_log", "log(%s)"); + elem["mu"] << lognormalLogName(m0, tool); + elem["sigma"] << lognormalLogName(k, tool); } - - return true; } -bool exportExponential(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportExponential(RooJSONFactoryWSTool *tool, RooExponential const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->variable().GetName(); - auto &c = pdf->coefficient(); - if (pdf->negateCoefficient()) { - elem["c"] << c.GetName(); + elem["x"] << pdf.variable().GetName(); + auto &c = pdf.coefficient(); + if (pdf.negateCoefficient()) { + writeReference(elem["c"], c); } else { elem["c"] << tool->exportTransformed(&c, "_exponential_inverted", "-%s"); } - - return true; } -bool exportMultiVarGaussian(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportMultiVarGaussian(RooJSONFactoryWSTool *, RooMultiVarGaussian const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["x"], pdf->xVec()); - RooJSONFactoryWSTool::fillSeq(elem["mean"], pdf->muVec()); - elem["covariances"].fill_mat(pdf->covarianceMatrix()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["x"], pdf.xVec()); + RooJSONFactoryWSTool::fillSeq(elem["mean"], pdf.muVec()); + elem["covariances"].fill_mat(pdf.covarianceMatrix()); } -bool exportTFnBinding(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportTFnBinding(RooJSONFactoryWSTool *, RooTFnBinding const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - - TString formula(pdf->function().GetExpFormula()); - formula.ReplaceAll("x", pdf->observables()[0].GetName()); - formula.ReplaceAll("y", pdf->observables()[1].GetName()); - formula.ReplaceAll("z", pdf->observables()[2].GetName()); - for (size_t i = 0; i < pdf->parameters().size(); ++i) { + checkExpressionRefersToNoPdf(pdf, pdf.observables()); + checkExpressionRefersToNoPdf(pdf, pdf.parameters()); + TString formula(pdf.function().GetExpFormula()); + formula.ReplaceAll("x", pdf.observables()[0].GetName()); + formula.ReplaceAll("y", pdf.observables()[1].GetName()); + formula.ReplaceAll("z", pdf.observables()[2].GetName()); + for (size_t i = 0; i < pdf.parameters().size(); ++i) { TString pname(TString::Format("[%d]", (int)i)); - formula.ReplaceAll(pname, pdf->parameters()[i].GetName()); + formula.ReplaceAll(pname, pdf.parameters()[i].GetName()); } elem["expression"] << formula.Data(); - return true; } -bool exportDerivative(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportDerivative(RooJSONFactoryWSTool *, RooDerivative const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - elem["x"] << pdf->getX().GetName(); - elem["function"] << pdf->getFunc().GetName(); - if (!pdf->getNset().empty()) { - RooJSONFactoryWSTool::fillSeq(elem["normalization"], pdf->getNset()); + elem["x"] << pdf.getX().GetName(); + elem["function"] << pdf.getFunc().GetName(); + if (!pdf.getNset().empty()) { + RooJSONFactoryWSTool::fillSeq(elem["normalization"], pdf.getNset()); } - elem["order"] << pdf->order(); - elem["eps"] << pdf->eps(); - return true; + elem["order"] << pdf.order(); + elem["eps"] << pdf.eps(); } -bool exportRealIntegral(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportRealIntegral(RooJSONFactoryWSTool *, RooRealIntegral const &integral, JSONNode &elem) { - auto *integral = static_cast(func); - elem["type"] << key; - std::string integrand = integral->integrand().GetName(); + std::string integrand = integral.integrand().GetName(); elem["integrand"] << integrand; - if (integral->intRange()) { - elem["domain"] << integral->intRange(); + if (integral.intRange()) { + elem["domain"] << integral.intRange(); } - RooJSONFactoryWSTool::fillSeq(elem["variables"], integral->intVars()); - if (RooArgSet const *funcNormSet = integral->funcNormSet()) { + RooJSONFactoryWSTool::fillSeq(elem["variables"], integral.intVars()); + if (RooArgSet const *funcNormSet = integral.funcNormSet()) { RooJSONFactoryWSTool::fillSeq(elem["normalization"], *funcNormSet); } - return true; } -bool exportFFTConvPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportFFTConvPdf(RooJSONFactoryWSTool *, RooFFTConvPdf const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - if (auto convFunc = pdf->getPdfConvVar()) { + if (auto convFunc = pdf.getPdfConvVar()) { elem["conv_func"] << convFunc->GetName(); } - elem["conv_var"] << pdf->getConvVar().GetName(); - elem["pdf1"] << pdf->getPdf1().GetName(); - elem["pdf2"] << pdf->getPdf2().GetName(); - elem["ipOrder"] << pdf->getInterpolationOrder(); - return true; + elem["conv_var"] << pdf.getConvVar().GetName(); + elem["pdf1"] << pdf.getPdf1().GetName(); + elem["pdf2"] << pdf.getPdf2().GetName(); + elem["ipOrder"] << pdf.getInterpolationOrder(); } -bool exportExtendPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportExtendPdf(RooJSONFactoryWSTool *, RooExtendPdf const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - if (auto rangeName = pdf->getRangeName()) { + if (auto rangeName = pdf.getRangeName()) { elem["range"] << rangeName->GetName(); } - elem["dist"] << pdf->pdf().GetName(); - elem["rate"] << pdf->getN().GetName(); - return true; + elem["dist"] << pdf.pdf().GetName(); + writeReference(elem["rate"], pdf.getN()); } -bool exportParamHistFunc(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportParamHistFunc(RooJSONFactoryWSTool *, ParamHistFunc const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["variables"], pdf->dataVars()); + RooJSONFactoryWSTool::fillSeq(elem["variables"], pdf.dataVars()); // One parameter per bin, in the order of HS3 (last axis fastest), not in ParamHistFunc's internal TH1 order. auto ¶msNode = elem["parameters"].set_seq(); - for (RooAbsReal *param : paramHistFuncParamsInBinOrder(*pdf)) { - paramsNode.append_child() << param->GetName(); + for (RooAbsReal *param : paramHistFuncParamsInBinOrder(pdf)) { + writeReference(paramsNode.append_child(), *param); } auto &observablesNode = elem["axes"].set_seq(); // axes have to be ordered to get consistent bin indices - for (auto *var : static_range_cast(pdf->dataVars())) { - RooJSONFactoryWSTool::exportAxis(observablesNode.append_child().set_map(), *var); + for (auto *var : static_range_cast(pdf.dataVars())) { + HS3::writeAxis(observablesNode.append_child(), HS3::toBinnedAxis(*var)); } - return true; } -bool exportSpline(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportSpline(RooJSONFactoryWSTool *, RooSpline const &rs, JSONNode &elem) { - auto const *rs = static_cast(func); - - elem["type"] << key; - - // Independent variable - elem["x"] << rs->x().GetName(); + elem["x"] << rs.x().GetName(); + elem["interpolation"] << (rs.order() == 5 ? "poly5" : "poly3"); + elem["logx"] << rs.logx(); + elem["logy"] << rs.logy(); - // Spline configuration - // Canonical algo for RooSpline - elem["interpolation"] << (rs->order() == 5 ? "poly5" : "poly3"); - elem["logx"] << rs->logx(); - elem["logy"] << rs->logy(); - - // Serialize knots as primitive arrays - TSpline const &sp = rs->spline(); + // The knots + TSpline const &sp = rs.spline(); auto &x0 = elem["x0"].set_seq(); auto &y0 = elem["y0"].set_seq(); - - const int np = sp.GetNp(); - for (int i = 0; i < np; ++i) { + for (int i = 0; i < sp.GetNp(); ++i) { double xk = 0.0; double yk = 0.0; sp.GetKnot(i, xk, yk); x0.append_child() << xk; y0.append_child() << yk; } - - return true; } -bool exportAddition(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportBernstein(RooJSONFactoryWSTool *, RooBernstein const &pdf, JSONNode &elem) { - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["summands"], static_cast(func)->list()); - return true; + writeReference(elem["x"], pdf.x()); + RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf.coefList()); } -bool exportArgus(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportLandau(RooJSONFactoryWSTool *, RooLandau const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["mass"], pdf->getM()); - writeReference(elem["resonance"], pdf->getM0()); - writeReference(elem["slope"], pdf->getC()); - writeReference(elem["power"], pdf->getP()); - return true; + writeReference(elem["x"], pdf.getX()); + writeReference(elem["mu"], pdf.getMean()); + writeReference(elem["sigma"], pdf.getSigma()); } -bool exportBernstein(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportProduct(RooJSONFactoryWSTool *, RooProduct const &prod, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->x()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["factors"], prod.realComponents()); + RooJSONFactoryWSTool::fillSeq(elem["factors"], prod.categoricalComponents()); } -bool exportBifurGauss(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportUniform(RooJSONFactoryWSTool *, RooUniform const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->getX()); - writeReference(elem["mean"], pdf->getMean()); - writeReference(elem["sigmaL"], pdf->getSigmaL()); - writeReference(elem["sigmaR"], pdf->getSigmaR()); - return true; + RooJSONFactoryWSTool::fillSeq(elem["x"], pdf.observables()); } -bool exportCBShape(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void importWrapperPdf(RooJSONFactoryWSTool *tool, const JSONNode &p) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["m"], pdf->getM()); - writeReference(elem["m0"], pdf->getM0()); - writeReference(elem["sigma"], pdf->getSigma()); - writeReference(elem["alpha"], pdf->getAlpha()); - writeReference(elem["n"], pdf->getN()); - return true; + std::string name(RooJSONFactoryWSTool::name(p)); + RooAbsReal *func = tool->requestArg(p, "function"); + // ROOT previously exported this key without an underscore. + const char *key = p.has_child("self_normalized") ? "self_normalized" : "selfnormalized"; + const bool selfNormalized = p.has_child(key) && p[key].val_bool(); + tool->wsEmplace(name, *func, selfNormalized); } -bool exportChebychev(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +void exportWrapperPdf(RooJSONFactoryWSTool *, RooWrapperPdf const &pdf, JSONNode &elem) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->x()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - return true; + // RooWrapperPdf has no accessor for the wrapped function, which is its only proxy. + auto const &funcProxy = dynamic_cast(*pdf.getProxy(0)); + elem["function"] << funcProxy.absArg()->GetName(); + if (pdf.selfNormalized()) { + elem["self_normalized"] << true; + } } -bool exportEffProd(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["pdf"], pdf->pdf()); - writeReference(elem["eff"], pdf->eff()); - return true; -} +/////////////////////////////////////////////////////////////////////////////////////////////////////// +// instantiate all importers and exporters +/////////////////////////////////////////////////////////////////////////////////////////////////////// -bool exportGamma(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->getX()); - writeReference(elem["gamma"], pdf->getGamma()); - writeReference(elem["beta"], pdf->getBeta()); - writeReference(elem["mu"], pdf->getMu()); - return true; -} +// A class whose constructor takes nothing but objects and lists of objects, each of which it has a getter for, is +// read and written field by field. A field describes a constructor argument: its key in the document, and the getter +// that gives what is written. -bool exportLandau(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->getX()); - writeReference(elem["mean"], pdf->getMean()); - writeReference(elem["sigma"], pdf->getSigma()); - return true; -} +// An argument that is one object: requested as a T, and written as a reference to it. +template +struct ObjectField { + const char *key; + Getter_t getter; -bool exportPowerSum(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["x"], pdf->base()); - RooJSONFactoryWSTool::fillSeq(elem["coefficients"], pdf->coefList()); - RooJSONFactoryWSTool::fillSeq(elem["exponents"], pdf->expList()); - return true; -} + T &request(RooJSONFactoryWSTool &tool, const JSONNode &p) const { return *tool.requestArg(p, key); } + template + void write(RooArg_t const &arg, JSONNode &elem) const + { + writeReference(elem[key], (arg.*getter)()); + } +}; -bool exportProdPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["factors"], static_cast(func)->pdfList()); - return true; -} +// An argument that is a list of objects: requested as a Collection_t of T, and written as a list of references. +template +struct ListField { + const char *key; + Getter_t getter; -bool exportProduct(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) -{ - auto *prod = static_cast(func); - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["factors"], prod->realComponents()); - RooJSONFactoryWSTool::fillSeq(elem["factors"], prod->categoricalComponents()); - return true; -} + Collection_t request(RooJSONFactoryWSTool &tool, const JSONNode &p) const + { + return tool.requestCollection(p, key); + } + template + void write(RooArg_t const &arg, JSONNode &elem) const + { + RooJSONFactoryWSTool::fillSeq(elem[key], (arg.*getter)()); + } +}; -bool exportProjectedPdf(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +template +ObjectField object(const char *key, Getter_t getter) { - auto *pdf = static_cast(func); - elem["type"] << key; - writeReference(elem["input_pdf"], pdf->intPdf()); - RooJSONFactoryWSTool::fillSeq(elem["observables"], pdf->intObs()); - return true; + return {key, getter}; } -bool exportUniform(RooJSONFactoryWSTool *, const RooAbsArg *func, JSONNode &elem, std::string const &key) +template +ListField list(const char *key, Getter_t getter) { - elem["type"] << key; - RooJSONFactoryWSTool::fillSeq(elem["x"], static_cast(func)->observables()); - return true; + return {key, getter}; } -bool importWrapperPdf(RooJSONFactoryWSTool *tool, const JSONNode &node) +// Register the importer and the exporter of the class RooArg_t, whose constructor takes the name, the title and then +// the fields in the given order. +template +void registerFields(const std::string &key, Fields_t... fields) { - if (node["type"].val() != "density_function_dist") - return false; - - auto name = RooJSONFactoryWSTool::name(node); - auto *func = tool->requestArg(node, "function"); - - bool selfNormalized = false; - - auto sn = node.find("self_normalized"); - // ROOT previously exported this key without an underscore. - if (!sn) - sn = node.find("selfnormalized"); - if (sn) - selfNormalized = sn->val_bool(); - - tool->wsEmplace(name, *func, selfNormalized); - return true; + auto importFields = [=](RooJSONFactoryWSTool *tool, const JSONNode &p) { + const std::string name = RooJSONFactoryWSTool::name(p); + // The braces guarantee that the fields are requested in their order. + tool->wsImport(RooArg_t{name.c_str(), name.c_str(), fields.request(*tool, p)...}); + }; + auto exportFields = [=](RooJSONFactoryWSTool *, RooArg_t const &arg, JSONNode &elem) { + (fields.write(arg, elem), ...); + }; + registerImporter(key, importFields); + registerExporter(key, exportFields); } -bool exportWrapperPdf(RooJSONFactoryWSTool *, const RooAbsArg *arg, JSONNode &node, std::string const &key) +// A deprecated type key is still read by the importers of its replacement, with a warning the first time. +void registerDeprecatedKey(const std::string &key, const std::string &replacement) { - auto const *pdf = dynamic_cast(arg); - if (!pdf) - return false; - - node["type"] << key; - - // Proxy name in RooWrapperPdf is "_func" / "func" depending on accessor/proxy export. - // Prefer a public accessor if one exists; otherwise inspect proxies as below. - auto const *funcProxy = dynamic_cast(pdf->getProxy(0)); - if (!funcProxy || !funcProxy->absArg()) - return false; - - node["function"] << funcProxy->absArg()->GetName(); - if (pdf->selfNormalized()) - node["self_normalized"] << true; - - return true; -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////// -// instantiate all importers and exporters -/////////////////////////////////////////////////////////////////////////////////////////////////////// - -// Adapters that wrap the plain import/export functions above into the -// RooFit::JSONIO::Importer/Exporter interface. The exporter also owns the HS3 -// type key, which is passed at registration time. -template -class FuncImporter : public RooFit::JSONIO::Importer { -public: - bool importArg(RooJSONFactoryWSTool *tool, const JSONNode &p) const override { return Func(tool, p); } -}; - -template -class FuncExporter : public RooFit::JSONIO::Exporter { -public: - FuncExporter(std::string key, bool autoExportDependants) - : _key{std::move(key)}, _autoExportDependants{autoExportDependants} - { - } - std::string const &key() const override { return _key; } - bool autoExportDependants() const override { return _autoExportDependants; } - bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *func, JSONNode &elem) const override - { - return Func(tool, func, elem, _key); - } - -private: - const std::string _key; - const bool _autoExportDependants; -}; - -template -void registerImporter(const std::string &key, bool topPriority = true) -{ - RooFit::JSONIO::registerImporter(key, std::make_unique>(), topPriority); -} - -template -void registerExporter(TClass const *cl, std::string key, bool topPriority = true, bool autoExportDependants = true) -{ - RooFit::JSONIO::registerExporter(cl, std::make_unique>(std::move(key), autoExportDependants), - topPriority); -} - -STATIC_EXECUTE([]() { - registerImporter("density_function_dist"); - registerImporter("rate_extended_dist"); - RooFit::JSONIO::registerImporter("crystalball_dist", std::make_unique()); - registerImporter("crystalball_doublesided_dist"); - RooFit::JSONIO::registerImporter("ARGUS_dist", std::make_unique("ARGUS_dist", "argus_dist")); - RooFit::JSONIO::registerImporter("binsampling", - std::make_unique("binsampling", "binsampling_dist")); - RooFit::JSONIO::registerImporter( - "mixture_resolution_model", std::make_unique("mixture_resolution_model", "mixture_model")); - RooFit::JSONIO::registerImporter("delta_resolution_model", std::make_unique( - "delta_resolution_model", "truth_model_function")); - RooFit::JSONIO::registerImporter("gauss_resolution_model", std::make_unique( - "gauss_resolution_model", "gauss_model_function")); - registerImporter("argus_dist", false); - registerImporter("ARGUS_dist", false); - registerImporter("bernstein_poly_dist", false); - registerImporter("bifurkated_gaussian_dist", false); - registerImporter("chebychev_dist", false); - registerImporter("efficiency_product_pdf_dist", false); - registerImporter("gamma_dist", false); - registerImporter("gaussian_dist", false); - registerImporter("normal_dist", false); - registerImporter("landau_dist", false); - registerImporter("power_sum_dist", false); - registerImporter("projected_dist", false); - registerImporter("uniform_dist", false); - registerImporter("product", false); - registerImporter("product_dist", false); - registerImporter("array_dist", false); - registerImporter("sum", false); - registerImporter("mixture_dist", false); - registerImporter("mixture_model", false); - registerImporter("combined_dist", false); - registerImporter("mixture_resolution_model", false); - registerImporter("binsampling_dist", false); - registerImporter("binsampling", false); - registerImporter>("binvolume", false); - registerImporter>("inverse_binvolume", false); - registerImporter>("legacy_exp_poly_dist", false); - registerImporter("exponential_dist", false); - registerImporter>("generic", false); - registerImporter>("generic_function", false); - registerImporter>("generic_dist", false); - registerImporter>("histogram", false); - registerImporter>("step", false); - registerImporter>("histogram_dist", false); - registerImporter("lognormal_dist", false); - registerImporter("multivariate_normal_dist", false); - registerImporter>("poisson_dist", false); - registerImporter>("poisson_continuous_dist", false); - registerImporter("decay_dist", false); - registerImporter("truth_model_function", false); - registerImporter("delta_resolution_model", false); - registerImporter("gauss_model_function", false); - registerImporter("gauss_resolution_model", false); - registerImporter>("polynomial_dist", false); - registerImporter>("polynomial", false); - registerImporter>("weighted_sum_dist", false); - registerImporter>("weighted_sum", false); - registerImporter("integral", false); - registerImporter("derivative", false); - registerImporter("fft_convolution_dist", false); - registerImporter("extend_pdf", false); - registerImporter("step", false); - registerImporter("spline", false); - - registerExporter(RooWrapperPdf::Class(), "density_function_dist"); - registerExporter>(RooAddPdf::Class(), "mixture_dist", false); - registerExporter>(RooAddModel::Class(), "mixture_model", false); - registerExporter(RooBinSamplingPdf::Class(), "binsampling_dist", false); - registerExporter(RooCrystalBall::Class(), "crystalball_doublesided_dist", false); - registerExporter(RooBinWidthFunction::Class(), "binvolume", false); - registerExporter>(RooLegacyExpPoly::Class(), "legacy_exp_poly_dist", false); - registerExporter(RooExponential::Class(), "exponential_dist", false); - registerExporter>(RooFormulaVar::Class(), "generic", false); - registerExporter>(RooGenericPdf::Class(), "generic_dist", false); - registerExporter>(RooHistFunc::Class(), "step", false); - registerExporter>(RooHistPdf::Class(), "histogram_dist", false); - registerExporter(RooLognormal::Class(), "lognormal_dist", false); - registerExporter(RooGaussian::Class(), "gaussian_dist", true); - registerExporter(RooMultiVarGaussian::Class(), "multivariate_normal_dist", false); - registerExporter(RooPoisson::Class(), "poisson_dist", false); - registerExporter(RooDecay::Class(), "decay_dist", false, /*autoExportDependants=*/false); - registerExporter(RooTruthModel::Class(), "truth_model_function", false); - registerExporter(RooGaussModel::Class(), "gauss_model_function", false); - registerExporter>(RooPolynomial::Class(), "polynomial_dist", false); - registerExporter>(RooPolyVar::Class(), "polynomial", false); - registerExporter>(RooRealSumFunc::Class(), "weighted_sum", false); - registerExporter>(RooRealSumPdf::Class(), "weighted_sum_dist", false); - registerExporter(RooTFnBinding::Class(), "generic", false); - registerExporter(RooRealIntegral::Class(), "integral", false); - registerExporter(RooDerivative::Class(), "derivative", false); - registerExporter(RooFFTConvPdf::Class(), "fft_convolution_dist", false); - registerExporter(RooExtendPdf::Class(), "rate_extended_dist", false); - registerExporter(ParamHistFunc::Class(), "step", false); - registerExporter(RooSpline::Class(), "spline", false); - registerExporter(RooAddition::Class(), "sum", false); - registerExporter(RooArgusBG::Class(), "argus_dist", false); - registerExporter(RooBernstein::Class(), "bernstein_poly_dist", false); - registerExporter(RooBifurGauss::Class(), "bifurkated_gaussian_dist", false); - registerExporter(RooCBShape::Class(), "crystalball_dist", false); - registerExporter(RooChebychev::Class(), "chebychev_dist", false); - registerExporter(RooEffProd::Class(), "efficiency_product_pdf_dist", false); - registerExporter(RooGamma::Class(), "gamma_dist", false); - registerExporter(RooLandau::Class(), "landau_dist", false); - registerExporter(RooPowerSum::Class(), "power_sum_dist", false); - registerExporter(RooProdPdf::Class(), "product_dist", false); - registerExporter(RooProduct::Class(), "product", false); - registerExporter(RooProjectedPdf::Class(), "projected_dist", false); - registerExporter(RooUniform::Class(), "uniform_dist", false); -}); + registerImporter(key, [=](RooJSONFactoryWSTool *tool, const JSONNode &p) { + static std::set warned; + if (warned.insert(key).second) { + RooJSONFactoryWSTool::warning("RooFitHS3: the type \"" + key + "\" is deprecated, please use \"" + + replacement + "\" instead."); + } + auto const &importers = RooFit::JSONIO::importers().at(replacement); + return std::any_of(importers.begin(), importers.end(), + [&](auto const &importer) { return importer->importArg(tool, p); }); + }); +} + +// Registers the importers and exporters when the library is loaded. +const bool registered = []() { + registerType("bernstein_poly_dist", importBernstein, exportBernstein); + registerType("binsampling_dist", importBinSamplingPdf, exportBinSamplingPdf); + registerType("binvolume", importBinWidthFunction, exportBinWidthFunction); + registerType("crystalball_doublesided_dist", importCrystalBallDoubleSided, exportCrystalBall); + registerType("decay_dist", importDecay, exportDecay, /*autoExportDependants=*/false); + registerType("density_function_dist", importWrapperPdf, exportWrapperPdf); + registerType("derivative", importDerivative, exportDerivative); + registerType("exponential_dist", importExponential, exportExponential); + registerType("fft_convolution_dist", importFFTConvPdf, exportFFTConvPdf); + registerType("gaussian_dist", importGaussian, exportGaussian); + registerType("generic", importFormulaArg, exportFormulaArg); + registerType("generic_dist", importFormulaArg, exportFormulaArg); + registerType("histogram_dist", importHist, exportHist); + registerType("integral", importRealIntegral, exportRealIntegral); + registerType("landau_dist", importLandau, exportLandau); + registerType("legacy_exp_poly_dist", importPolynomial, exportPolynomial); + registerType("lognormal_dist", importLogNormal, exportLogNormal); + registerType("mixture_dist", importAddPdf, exportAddPdf); + registerType("mixture_model", importAddModel, exportAddPdf); + registerType("multivariate_normal_dist", importMultiVarGaussian, exportMultiVarGaussian); + registerType("poisson_dist", importPoisson, exportPoisson); + registerType("polynomial", importPolynomial, exportPolynomial); + registerType("polynomial_dist", importPolynomial, exportPolynomial); + registerType("product", importProduct, exportProduct); + registerType("rate_extended_dist", importExtendPdf, exportExtendPdf); + registerType("spline", importSpline, exportSpline); + // A "step" with "data" is a histogram, otherwise it is a ParamHistFunc. + registerType("step", importHist, exportHist); + registerType("step", importParamHistFunc, exportParamHistFunc); + registerType("uniform_dist", importUniform, exportUniform); + registerType("weighted_sum_dist", importRealSumPdf, exportRealSumPdf); + + // The classes that are read and written field by field. + registerFields("argus_dist", object("mass", &RooArgusBG::getM), object("resonance", &RooArgusBG::getM0), + object("slope", &RooArgusBG::getC), object("power", &RooArgusBG::getP)); + registerFields("bifurkated_gaussian_dist", object("x", &RooBifurGauss::getX), + object("mean", &RooBifurGauss::getMean), object("sigmaL", &RooBifurGauss::getSigmaL), + object("sigmaR", &RooBifurGauss::getSigmaR)); + // A "crystalball_dist" with the fields of two tails is the double-sided Crystal Ball of documents before HS3 0.3. + registerImporter("crystalball_dist", importLegacyDoubleSidedCrystalBall); + registerFields("crystalball_dist", object("m", &RooCBShape::getM), object("m0", &RooCBShape::getM0), + object("sigma", &RooCBShape::getSigma), object("alpha", &RooCBShape::getAlpha), + object("n", &RooCBShape::getN)); + registerFields("chebychev_dist", object("x", &RooChebychev::x), + list("coefficients", &RooChebychev::coefList)); + registerFields("efficiency_product_pdf_dist", object("pdf", &RooEffProd::pdf), + object("eff", &RooEffProd::eff)); + registerFields("gamma_dist", object("x", &RooGamma::getX), object("gamma", &RooGamma::getGamma), + object("beta", &RooGamma::getBeta), object("mu", &RooGamma::getMu)); + registerFields("gauss_model_function", object("x", &RooGaussModel::convVar), + object("mean", &RooGaussModel::getMean), object("sigma", &RooGaussModel::getSigma)); + registerFields("power_sum_dist", object("x", &RooPowerSum::base), + list("coefficients", &RooPowerSum::coefList), list("exponents", &RooPowerSum::expList)); + registerFields("product_dist", list("factors", &RooProdPdf::pdfList)); + registerFields("projected_dist", object("input_pdf", &RooProjectedPdf::intPdf), + list("observables", &RooProjectedPdf::intObs)); + registerFields("sum", list("summands", &RooAddition::list)); + registerFields("truth_model_function", object("x", &RooTruthModel::convVar)); + registerFields("weighted_sum", list("samples", &RooRealSumFunc::funcList), + list("coefficients", &RooRealSumFunc::coefList)); + + // A RooTFnBinding is written as a generic formula. + registerExporter("generic", exportTFnBinding); + + // The types that are only read: the ones that RooFit writes in another way, and other names of the types above. + registerImporter("array_dist", importArrayDist); + registerImporter("combined_dist", importCombinedDist); + registerImporter("extend_pdf", importExtendPdf); + registerImporter("generic_function", importFormulaArg); + registerImporter("histogram", importHist); + registerImporter("inverse_binvolume", importBinWidthFunction); + registerImporter("normal_dist", importGaussian); + registerImporter("poisson_continuous_dist", importPoisson); + registerDeprecatedKey("ARGUS_dist", "argus_dist"); + registerDeprecatedKey("binsampling", "binsampling_dist"); + registerDeprecatedKey("delta_resolution_model", "truth_model_function"); + registerDeprecatedKey("gauss_resolution_model", "gauss_model_function"); + registerDeprecatedKey("mixture_resolution_model", "mixture_model"); + return true; +}(); } // namespace -std::optional arrayComponentInfo(RooAbsPdf const &pdf, std::string const *observable) +std::optional +arrayComponentInfo(RooAbsPdf const &pdf, std::string const *observable, RooJSONFactoryWSTool *tool) { - auto component = arrayComponentOf(pdf, observable); - if (!component) { + ArrayComponentInfo out; + RooAbsReal const *x = nullptr; + if (auto const *poisson = dynamic_cast(&pdf)) { + out.type = poisson->getNoRounding() ? "poisson_continuous_dist" : "poisson_dist"; + x = &poisson->getX(); + out.fields = {{"mean", HS3::toNameOrNumber(poisson->getMean())}}; + } else if (auto const *gauss = dynamic_cast(&pdf)) { + out.type = "gaussian_dist"; + x = &gauss->getX(); + RooAbsReal const *mean = &gauss->getMean(); + // A Gaussian constraint built by HistFactory has the parameter as x and the global observable as mean. The + // Gaussian is symmetric in the two, so it is written with the observable as x where the observable is known. + if (observable && *observable == mean->GetName() && !isLiteralConstVar(*mean)) { + std::swap(x, mean); + } + out.fields = {{"mean", HS3::toNameOrNumber(*mean)}, {"sigma", HS3::toNameOrNumber(gauss->getSigma())}}; + } else if (auto const *lognormal = dynamic_cast(&pdf)) { + out.type = "lognormal_dist"; + x = &lognormal->getX(); + if (lognormal->useStandardParametrization()) { + out.fields = {{"mu", HS3::toNameOrNumber(lognormal->getMedian())}, + {"sigma", HS3::toNameOrNumber(lognormal->getShapeK())}}; + } else { + out.fields = {{"mu", lognormalLogName(lognormal->getMedian(), tool)}, + {"sigma", lognormalLogName(lognormal->getShapeK(), tool)}}; + } + } else { + return std::nullopt; + } + // The observable of a component must be named: the observables are pairwise distinct and are paired with data. + if (isLiteralConstVar(*x)) { return std::nullopt; } - return ArrayComponentInfo{component->type, component->x}; + out.observable = x->GetName(); + return out; } -void writeArrayDist(JSONNode &elem, std::string const &name, std::vector const &components, - std::map const &observableOf) +void writeArrayDist(RooJSONFactoryWSTool &tool, JSONNode &elem, std::string const &name, + std::vector const &components) { auto error = [&name](std::string const &message) { RooJSONFactoryWSTool::error("cannot write the array distribution '" + name + "': " + message); @@ -2271,22 +1739,21 @@ void writeArrayDist(JSONNode &elem, std::string const &name, std::vector comps; + std::vector comps; std::set observables; for (RooAbsPdf const *pdf : components) { - auto recorded = observableOf.find(pdf->GetName()); - auto comp = arrayComponentOf(*pdf, recorded != observableOf.end() ? &recorded->second : nullptr); + auto comp = arrayComponentInfo(*pdf, tool.constraintObservable(pdf->GetName()), &tool); if (!comp) { error("'" + std::string(pdf->GetName()) + - "' cannot be a component, only RooPoisson, RooGaussian and RooLognormal (standard parametrization) " - "over a named observable are supported"); + "' cannot be a component, only RooPoisson, RooGaussian and RooLognormal over a named observable " + "are supported"); } if (!comps.empty() && comp->type != comps.front().type) { error("the components '" + std::string(components.front()->GetName()) + "' (" + comps.front().type + ") and '" + pdf->GetName() + "' (" + comp->type + ") have different types"); } - if (!observables.insert(comp->x).second) { - error("two components have the observable '" + comp->x + "'"); + if (!observables.insert(comp->observable).second) { + error("two components have the observable '" + comp->observable + "'"); } comps.push_back(std::move(*comp)); } @@ -2294,7 +1761,7 @@ void writeArrayDist(JSONNode &elem, std::string const &name, std::vector -#include - #include #include @@ -28,45 +26,31 @@ ImportMap &importers() namespace { -auto &exportersToAdd() -{ - static std::map>> toAdd; - return toAdd; -} +// An exporter that was registered by the name of its class, for which there might have been no TClass yet. +struct ExporterByName { + std::string className; + std::unique_ptr exporter; + bool topPriority = true; +}; -ExportMap &exportersImpl() +std::vector &exportersByName() { - static ExportMap _exporters; - return _exporters; + static std::vector _exportersByName; + return _exportersByName; } } // namespace ExportMap &exporters() { - // If there are exporters to be added, do this now. - for (auto &item : exportersToAdd()) { - TClass *klass = TClass::GetClass(item.first.c_str()); - auto &exporters = exportersImpl()[klass]; // registered exporters so far - auto &toAdd = item.second; // exporters to add - - // Find the nullptr separator - auto nullIt = std::find(toAdd.begin(), toAdd.end(), nullptr); - - if (nullIt == toAdd.end()) { - throw std::runtime_error("toAdd does not contain nullptr separator"); - } - - // Move elements after nullptr to the back - exporters.insert(exporters.end(), std::make_move_iterator(nullIt + 1), std::make_move_iterator(toAdd.end())); - - // Move elements before nullptr to the front - exporters.insert(exporters.begin(), std::make_move_iterator(toAdd.begin()), std::make_move_iterator(nullIt)); - - toAdd.clear(); + static ExportMap _exporters; + // The exporters that were registered by class name are registered with their TClass now, in the same order. + std::vector byName; + byName.swap(exportersByName()); + for (auto &item : byName) { + registerExporter(TClass::GetClass(item.className.c_str()), std::move(item.exporter), item.topPriority); } - exportersToAdd().clear(); - return exportersImpl(); + return _exporters; } bool registerImporter(const std::string &key, std::unique_ptr f, bool topPriority) @@ -85,14 +69,7 @@ bool registerExporter(const TClass *key, std::unique_ptr f, bool bool registerExporter(const std::string &key, std::unique_ptr f, bool topPriority) { - auto &vec = exportersToAdd()[key]; - // The vector starts out with just a nullptr separator. Elements before it - // will be added to the top of the exporter queue, and the elements after - // get appended to the end. - if (vec.empty()) { - vec.emplace_back(nullptr); - } - vec.insert(topPriority ? vec.begin() : vec.end(), std::move(f)); + exportersByName().push_back({key, std::move(f), topPriority}); return true; } diff --git a/roofit/hs3/src/JSONIOUtils.cxx b/roofit/hs3/src/JSONIOUtils.cxx index d57e7ff202af3..96f7ac868cc90 100644 --- a/roofit/hs3/src/JSONIOUtils.cxx +++ b/roofit/hs3/src/JSONIOUtils.cxx @@ -1,6 +1,5 @@ #include "JSONIOUtils.h" -#include #include #include #include @@ -13,6 +12,8 @@ #include #include +#include +#include #include #include #include @@ -21,22 +22,19 @@ using RooFit::Detail::JSONNode; -// Parse the entire string as a number (integer or floating-point). The output is only valid if that succeeds. -ToNumberOutput toNumber(const std::string &str) +std::optional toNumber(const std::string &str) { // std::from_chars for floating-point types is not portably available on all // platforms ROOT supports, so we rely on the stream extraction. std::istringstream stream(str); - ToNumberOutput output; - output.valid = (stream >> output.value) && stream.eof(); - return output; + double value = 0.0; + return (stream >> value) && stream.eof() ? std::optional{value} : std::nullopt; } // True if `arg` is a RooConstVar whose name is a plain number (i.e. a literal constant). bool isLiteralConstVar(RooAbsArg const &arg) { - bool isRooConstVar = dynamic_cast(&arg); - return isRooConstVar && toNumber(arg.GetName()).valid; + return dynamic_cast(&arg) && toNumber(arg.GetName()); } // Write a reference to `arg`: the value if it is a literal constant, and the name otherwise. @@ -67,57 +65,86 @@ std::string removeSuffix(std::string_view str, std::string_view suffix) return out; } -// Writes a regular binning with "min", "max" and "nbins", and an irregular one with its "edges" only, since HS3 0.3 -// forbids "min" and "max" alongside them. -void writeAxisBinning(JSONNode &node, const RooAbsBinning &binning) +namespace { + +constexpr std::string_view literalPrefix = "const_"; + +// The shortest decimal representation of a value that is read back as the same value. +std::string shortestNumber(double value) +{ + char buffer[32]; + for (int precision = 1; precision <= 17; ++precision) { + std::snprintf(buffer, sizeof(buffer), "%.*g", precision, value); + if (std::strtod(buffer, nullptr) == value) { + break; + } + } + return buffer; +} + +// The number that the identifier of a literal constant spells, or an empty string if `name` is no such identifier. +std::string numberOfLiteralIdentifier(std::string_view name) { - if (binning.isUniform()) { - node["min"] << binning.lowBound(); - node["max"] << binning.highBound(); - node["nbins"] << binning.numBins(); - return; + if (!startsWith(name, literalPrefix) || name.size() == literalPrefix.size()) { + return ""; + } + std::string number; + for (char c : name.substr(literalPrefix.size())) { + if (c == 'p') { + number += '.'; + } else if (c == 'm') { + number += '-'; + } else if (std::isdigit(static_cast(c)) || c == 'e') { + number += c; + } else { + return ""; + } } + char *end = nullptr; + std::strtod(number.c_str(), &end); + return end == number.c_str() + number.size() ? number : ""; +} + +} // namespace - auto &edges = node["edges"].set_seq(); - edges.append_child() << binning.binLow(0); - for (int i = 0; i < binning.numBins(); ++i) { - edges.append_child() << binning.binHigh(i); +std::string transformedArgument(RooAbsReal const ¶m) +{ + if (!isLiteralConstVar(param)) { + return param.GetName(); } + // In parentheses if negative, such that "-%s" doesn't become "--1". + const std::string number = shortestNumber(param.getVal()); + return number[0] == '-' ? "(" + number + ")" : number; } -// Parse the leading "MAJOR.MINOR" of an HS3 version string like "0.2", "0.2.9" or "0.3". -bool parseHS3Version(std::string const &version, int &major, int &minor) +std::string transformedBaseName(RooAbsReal const ¶m) { - std::size_t pos = 0; - auto readNumber = [&](int &out) { - const std::size_t begin = pos; - while (pos < version.size() && std::isdigit(static_cast(version[pos]))) { - ++pos; - } - // Reject empty and absurdly long numbers, which std::stoi could not represent. - if (pos == begin || pos - begin > 9) { - return false; + if (!isLiteralConstVar(param)) { + return param.GetName(); + } + std::string out{literalPrefix}; + for (char c : shortestNumber(param.getVal())) { + if (c == '.') { + out += 'p'; + } else if (c == '-') { + out += 'm'; + } else if (c != '+') { + out += c; } - out = std::stoi(version.substr(begin, pos - begin)); - return true; - }; - if (!readNumber(major) || pos >= version.size() || version[pos] != '.') { - return false; } - ++pos; - return readNumber(minor); + return out; } -// Whether an HS3 version predates `major.minor`. Versions that can't be parsed count as old. That includes the empty -// version of objects that are imported without a document. -bool isHS3VersionBefore(std::string const &version, int major, int minor) +bool isLiteralIdentifier(std::string_view name) { - int versionMajor = 0; - int versionMinor = 0; - if (!parseHS3Version(version, versionMajor, versionMinor)) { - return true; - } - return std::make_pair(versionMajor, versionMinor) < std::make_pair(major, minor); + return !numberOfLiteralIdentifier(name).empty(); +} + +std::string transformedOriginal(std::string_view name, std::string_view suffix) +{ + const std::string base = removeSuffix(name, suffix); + const std::string number = numberOfLiteralIdentifier(base); + return number.empty() ? base : number; } namespace { @@ -195,10 +222,10 @@ RooArgList toParamHistFuncOrder(RooArgList const ¶msInBinOrder, RooAbsCollec return out; } -void checkParamHistFuncOrderIsDefined(std::string const &hs3Version, RooAbsCollection const &vars, +void checkParamHistFuncOrderIsDefined(RooFit::HS3::Version const &version, RooAbsCollection const &vars, std::string const &context) { - if (vars.size() >= 2 && isHS3VersionBefore(hs3Version, 0, 3)) { + if (vars.size() >= 2 && version < RooFit::HS3::Version{0, 3}) { RooJSONFactoryWSTool::error(context + " has a multi-dimensional binning. Documents before HS3 0.3 don't define the order " "of its per-bin parameters (RooFit wrote them in TH1 order, while the bin " @@ -304,52 +331,3 @@ std::string selectionLabel(RooAbsCategory const &indexCat, std::vector -readSelectionColumns(RooWorkspace const &ws, JSONNode const &node, const char *field, std::string const &context) -{ - auto const *list = node.find(field); - if (!list || !list->is_seq() || list->num_children() == 0) { - RooJSONFactoryWSTool::error("\"" + std::string(field) + "\" of " + context + - " must be a non-empty list of names of categorical axes"); - } - std::vector columns; - for (JSONNode const &child : list->children()) { - if (child.is_container() || child.val().empty()) { - RooJSONFactoryWSTool::error("\"" + std::string(field) + "\" of " + context + - " must be a non-empty list of names of categorical axes"); - } - RooCategory *cat = ws.cat(child.val()); - if (!cat) { - RooJSONFactoryWSTool::error("the column '" + child.val() + "' of " + context + - " is not declared by a categorical axis of a domain"); - } - if (std::find(columns.begin(), columns.end(), cat) != columns.end()) { - RooJSONFactoryWSTool::error("the column '" + child.val() + "' of " + context + " is listed twice"); - } - columns.push_back(cat); - } - return columns; -} - -std::vector -readSelectionKey(JSONNode const &component, std::vector const &columns, std::string const &context) -{ - auto const *keyNode = component.find("key"); - if (!keyNode || !keyNode->is_seq() || keyNode->num_children() != columns.size()) { - RooJSONFactoryWSTool::error("every component of " + context + " must have a \"key\" with " + - std::to_string(columns.size()) + " labels, one per column"); - } - std::vector key; - std::size_t i = 0; - for (JSONNode const &label : keyNode->children()) { - if (label.is_container() || !columns[i]->hasLabel(label.val())) { - RooJSONFactoryWSTool::error("the label '" + (label.is_container() ? std::string("") : label.val()) + - "' in position " + std::to_string(i) + " of a key of " + context + - " is not one of the categories of '" + columns[i]->GetName() + "'"); - } - key.push_back(label.val()); - ++i; - } - return key; -} diff --git a/roofit/hs3/src/JSONIOUtils.h b/roofit/hs3/src/JSONIOUtils.h index b96d556dc2bd3..3f0dee1feb0cf 100644 --- a/roofit/hs3/src/JSONIOUtils.h +++ b/roofit/hs3/src/JSONIOUtils.h @@ -6,15 +6,26 @@ #include #include #include +#include +#include #include #include +#include + +#include "HS3Document.h" class RooAbsArg; class RooAbsBinning; +class RooAbsCategory; class RooAbsCollection; class RooAbsPdf; class RooAbsReal; class RooArgList; +class RooArgSet; +class RooCategory; +class RooDataHist; +class RooJSONFactoryWSTool; +class RooWorkspace; class ParamHistFunc; // A RooProdPdf with this attribute represents an HS3 `array_dist`: a product of one-dimensional distributions of one @@ -22,54 +33,73 @@ class ParamHistFunc; // observable. inline constexpr const char *arrayDistTag = "hs3_array_dist"; -class RooAbsBinning; -class RooAbsCategory; -class RooAbsCollection; -class RooAbsReal; -class RooArgList; -class RooArgSet; -class RooCategory; -class RooDataHist; -class RooWorkspace; -class ParamHistFunc; +// An object with this attribute is a helper: RooFit created it to represent something of an HS3 document that it has +// no class for, e.g. the RooProdPdf that holds the components of an array distribution together, or the parts of a +// composite structure that stands for one HS3 object. A helper is not a statement of the user about the model, so: +// - it is only written if another object that is written refers to it by name, or if the HS3 object that it stands +// for is written (see isSkippedHelper()), but not just because it is in the workspace; +// - a helper that is not written is ignored wherever the export looks at the users of an object, e.g. to decide +// whether the object is a top-level object or whether a constraint can be a component of an array. +// The attribute is set by the importers and is not written to the document. Objects created by users don't have it +// and are always written. +inline constexpr const char *helperObjectTag = "hs3_helper"; // True if `arg` is a RooConstVar whose name is a plain number (i.e. a literal constant). bool isLiteralConstVar(RooAbsArg const &arg); -// The HS3 type and the observable of `pdf` as a component of an array distribution (RooPoisson, RooGaussian or -// RooLognormal in the standard parametrization, over a named observable), or std::nullopt if `pdf` cannot be a -// component. +// Whether the export of the workspace `ws` does not write `arg`: a helper (see helperObjectTag) that nothing written +// refers to, i.e. it has no clients in the workspace other than helpers that are skipped themselves and no ModelConfig +// names it; a helper that stands for an HS3 object of its own (an array distribution) and that nothing in the +// workspace uses is written as that object. The property is weakly transitive: any other object whose clients in the +// workspace are all skipped, and that no ModelConfig names, is skipped as well, so that what only a skipped helper +// uses is not written either. An array distribution that is skipped dissolves into its components, which are written +// on their own: it doesn't count as a user of them. Such an object is treated as if it were not in the workspace. +bool isSkippedHelper(RooAbsArg const &arg, RooWorkspace const &ws); + +// The HS3 type, the observable and the other fields of `pdf` as a component of an array distribution (RooPoisson, +// RooGaussian or RooLognormal, over a named observable), or std::nullopt if `pdf` cannot be a component. The fields +// are the ones of the exporter of its class. // If `observable` names the mean of a RooGaussian, the Gaussian is written with x and mean swapped (see -// RooJSONFactoryWSTool::setConstraintObservable()). +// RooJSONFactoryWSTool::setConstraintObservable()). The functions that a field names, e.g. for a RooLognormal that is +// not in the standard parametrization, are only written if there is a `tool`; without one, this only describes the +// component. struct ArrayComponentInfo { std::string type; std::string observable; + std::vector> fields; }; -std::optional arrayComponentInfo(RooAbsPdf const &pdf, std::string const *observable = nullptr); - -// Write the array distribution `name` with the given components into `elem`. `observableOf` gives the observable -// recorded for a component by name, if any. -void writeArrayDist(RooFit::Detail::JSONNode &elem, std::string const &name, - std::vector const &components, - std::map const &observableOf); +std::optional +arrayComponentInfo(RooAbsPdf const &pdf, std::string const *observable = nullptr, RooJSONFactoryWSTool *tool = nullptr); -struct ToNumberOutput { - bool valid = false; - double value = 0.0; -}; +// Write the array distribution `name` with the given components into `elem`. The `tool` knows the observables that +// are recorded for the components, and writes the functions that they need besides, e.g. for a RooLognormal that is +// not in the standard parametrization. +void writeArrayDist(RooJSONFactoryWSTool &tool, RooFit::Detail::JSONNode &elem, std::string const &name, + std::vector const &components); -ToNumberOutput toNumber(const std::string &str); +// The number that a string spells as a whole, if it does. +std::optional toNumber(const std::string &str); void writeReference(RooFit::Detail::JSONNode &node, RooAbsArg const &arg); bool startsWith(std::string_view str, std::string_view prefix); bool endsWith(std::string_view str, std::string_view suffix); std::string removeSuffix(std::string_view str, std::string_view suffix); -void writeAxisBinning(RooFit::Detail::JSONNode &node, const RooAbsBinning &binning); -// The key of a fixed domain axis (HS3 #131): the parameter only takes this value, and RooFit reads it as a -// RooConstVar. -inline constexpr const char *fixedValueKey = "value"; -bool parseHS3Version(std::string const &version, int &major, int &minor); -bool isHS3VersionBefore(std::string const &version, int major, int minor); +// Some RooFit classes are parametrized differently than their HS3 distribution, e.g. RooLognormal by default. They +// are written with a function of the parameter (RooJSONFactoryWSTool::exportTransformed()), which is named after the +// parameter with a suffix that says what the function is. A literal constant has no name that could be part of an +// identifier ("1.2", "-3", "1e-05"), so its value is spelled as one: "const_1p2", "const_m3", "const_1em05". +// +// What a transformed function is named after: the name of the parameter, or the identifier for the value of a +// literal constant. +std::string transformedBaseName(RooAbsReal const ¶m); +// What a transformed function is a function of: the name of the parameter, or the value of a literal constant with +// as many digits as it takes to read it back unchanged (in parentheses if it is negative). +std::string transformedArgument(RooAbsReal const ¶m); +// The inverse: what to create the RooFit object from, given the name of a transformed function with this suffix. This +// is the name of the parameter, or the value of a literal constant as a string. +std::string transformedOriginal(std::string_view name, std::string_view suffix); +// Whether a name is the identifier for the value of a literal constant. +bool isLiteralIdentifier(std::string_view name); // A RooHistFunc or RooHistPdf is evaluated with its own observables (`funcObs`), which are mapped by position onto // the variables of its histogram (`histObs`) and need not have the same names, e.g. after a RooCustomizer replaced @@ -85,7 +115,7 @@ std::vector paramHistFuncParamsInBinOrder(ParamHistFunc const &fun RooArgList toParamHistFuncOrder(RooArgList const ¶msInBinOrder, RooAbsCollection const &vars); // Documents before HS3 0.3 wrote the parameters of multi-dimensional ParamHistFuncs in TH1 order. No such documents // are known; they are rejected instead of being guessed. -void checkParamHistFuncOrderIsDefined(std::string const &hs3Version, RooAbsCollection const &vars, +void checkParamHistFuncOrderIsDefined(RooFit::HS3::Version const &version, RooAbsCollection const &vars, std::string const &context); // Selection keys (HS3 "combined" data sets and "combined_dist"): a key is one label per column, and the columns are @@ -105,12 +135,79 @@ std::string selectionLabel(RooAbsCategory const &indexCat, std::vector const &key); // A key as written in the document, for messages: ["a", "b"]. std::string selectionKeyToString(std::vector const &key); -// Read the columns of a combined distribution or data set from the list `field` of `node`: each entry must be the -// name of a category in the workspace, i.e. declared by a categorical axis of a domain. -std::vector readSelectionColumns(RooWorkspace const &ws, RooFit::Detail::JSONNode const &node, - const char *field, std::string const &context); -// Read and validate the "key" of a component: one label per column, each one of the categories of its column. -std::vector readSelectionKey(RooFit::Detail::JSONNode const &component, - std::vector const &columns, std::string const &context); + +// Adapters that wrap plain import and export functions into the RooFit::JSONIO::Importer and Exporter interface, for +// the importers and exporters that RooFit registers itself. + +// An import function that returns nothing always reads the object. One that returns a bool can decline by returning +// false, so that the next importer of the type key is tried. +template +class FuncImporter : public RooFit::JSONIO::Importer { +public: + explicit FuncImporter(ImportFunc func) : _func{func} {} + bool importArg(RooJSONFactoryWSTool *tool, const RooFit::Detail::JSONNode &p) const override + { + if constexpr (std::is_void_v) { + _func(tool, p); + return true; + } else { + return _func(tool, p); + } + } + +private: + const ImportFunc _func; +}; + +template +using ExportFunc = void (*)(RooJSONFactoryWSTool *, RooArg_t const &, RooFit::Detail::JSONNode &); + +// The exporter of the class RooArg_t writes the HS3 type key that the export function is registered with. A function +// overwrites it if the type depends on the object. +template +class FuncExporter : public RooFit::JSONIO::Exporter { +public: + FuncExporter(Func func, std::string key, bool autoExportDependants) + : _func{func}, _key{std::move(key)}, _autoExportDependants{autoExportDependants} + { + } + std::string const &key() const override { return _key; } + bool autoExportDependants() const override { return _autoExportDependants; } + bool exportObject(RooJSONFactoryWSTool *tool, const RooAbsArg *arg, RooFit::Detail::JSONNode &elem) const override + { + elem["type"] << _key; + _func(tool, static_cast(*arg), elem); + return true; + } + +private: + const Func _func; + const std::string _key; + const bool _autoExportDependants; +}; + +// The importers of a type key and the exporters of a class are tried in the order of their registration. +template +void registerImporter(const std::string &key, ImportFunc func) +{ + RooFit::JSONIO::registerImporter(key, std::make_unique>(func), false); +} + +template +void registerExporter(std::string key, Func func, bool autoExportDependants = true) +{ + RooFit::JSONIO::registerExporter( + RooArg_t::Class(), std::make_unique>(func, std::move(key), autoExportDependants), + false); +} + +// A class that is read and written with the same type key. +template +void registerType(const std::string &key, ImportFunc importFunc, ExportFunc exportFunc, + bool autoExportDependants = true) +{ + registerImporter(key, importFunc); + registerExporter(key, exportFunc, autoExportDependants); +} #endif diff --git a/roofit/hs3/src/JSONInterface.cxx b/roofit/hs3/src/JSONInterface.cxx index 682010cdb8ec6..e17de1a07adae 100644 --- a/roofit/hs3/src/JSONInterface.cxx +++ b/roofit/hs3/src/JSONInterface.cxx @@ -17,9 +17,12 @@ #include #include +#include #include +#include #include #include +#include #include #include @@ -80,21 +83,20 @@ nlohmann::json parse(std::string const &input) value.type_name()); } -// To check whether it's allowed to reset the type of an object. We allow -// this for nodes that have no type yet, or nodes with an empty string. -bool isResettingPossible(nlohmann::json const &node) +// Makes a value a map or an array. Only a value that has no type yet or is an empty string, which is what +// append_child() creates, can change its type. +void setContainerType(nlohmann::json &value, nlohmann::json::value_t type, std::string const &key) { - - if (node.type() == nlohmann::json::value_t::null) { - return true; + if (value.type() == type) { + return; } - - if (node.type() == nlohmann::json::value_t::string) { - if (node.get().empty()) { - return true; - } + if (!value.is_null() && !(value.is_string() && value.get_ref().empty())) { + throw std::runtime_error("cannot declare \"" + key + "\" to be of " + + (type == nlohmann::json::value_t::object ? "map" : "seq") + " - type, already of type " + + value.type_name()); } - return false; + // Not with braces, which would create an array with one element. + value = nlohmann::json(type); } } // namespace @@ -178,15 +180,6 @@ JSONNode &JSONNode::operator<<(bool b) return *this; } -const JSONNode &JSONNode::operator>>(std::string &v) const -{ - if (!json(_value).is_string()) { - throwTypeError(_key, "a string", json(_value)); - } - v = json(_value).get(); - return *this; -} - JSONNode &JSONNode::operator[](std::string const &k) { return _storage->makeNode(json(_value)[k], k); @@ -225,29 +218,13 @@ bool JSONNode::is_number() const JSONNode &JSONNode::set_map() { - if (json(_value).type() == nlohmann::json::value_t::object) - return *this; - - if (isResettingPossible(json(_value))) { - json(_value) = nlohmann::json::object(); - } else { - throw std::runtime_error("cannot declare \"" + this->key() + "\" to be of map - type, already of type " + - json(_value).type_name()); - } + setContainerType(json(_value), nlohmann::json::value_t::object, _key); return *this; } JSONNode &JSONNode::set_seq() { - if (json(_value).type() == nlohmann::json::value_t::array) - return *this; - - if (isResettingPossible(json(_value))) { - json(_value) = nlohmann::json::array(); - } else { - throw std::runtime_error("cannot declare \"" + this->key() + "\" to be of seq - type, already of type " + - json(_value).type_name()); - } + setContainerType(json(_value), nlohmann::json::value_t::array, _key); return *this; } @@ -278,11 +255,20 @@ std::string JSONNode::val() const switch (value.type()) { case nlohmann::json::value_t::string: return value.get(); case nlohmann::json::value_t::boolean: return value.get() ? "true" : "false"; - case nlohmann::json::value_t::number_integer: return std::to_string(value.get()); - case nlohmann::json::value_t::number_unsigned: return std::to_string(value.get()); + case nlohmann::json::value_t::number_integer: return std::to_string(value.get()); + case nlohmann::json::value_t::number_unsigned: return std::to_string(value.get()); case nlohmann::json::value_t::number_float: { + // The string has to read back as the same number. Start with the default precision of the streams, so that the + // numbers that six digits spell exactly keep their short spelling, and add digits as long as it takes. + const double number = value.get(); std::stringstream ss; - ss << value.get(); + for (int precision = 6; precision <= std::numeric_limits::max_digits10; ++precision) { + ss.str(""); + ss << std::setprecision(precision) << number; + if (std::strtod(ss.str().c_str(), nullptr) == number) { + break; + } + } return ss.str(); } default: @@ -329,11 +315,6 @@ std::optional JSONNode::try_val_bool() const return std::nullopt; } -bool JSONNode::has_key() const -{ - return !_key.empty(); -} - bool JSONNode::has_val() const { return json(_value).is_primitive(); @@ -380,12 +361,6 @@ JSONNode::const_children_view JSONNode::children() const return {const_child_iterator{list.data()}, const_child_iterator{list.data() + list.size()}}; } -std::ostream &operator<<(std::ostream &os, JSONNode const &s) -{ - s.writeJSON(os); - return os; -} - // JSONTree JSONTree::JSONTree(std::unique_ptr storage) : _storage{std::move(storage)} {} diff --git a/roofit/hs3/src/RooJSONFactoryWSTool.cxx b/roofit/hs3/src/RooJSONFactoryWSTool.cxx index 45b5a0d99f43c..7717db16b24bc 100644 --- a/roofit/hs3/src/RooJSONFactoryWSTool.cxx +++ b/roofit/hs3/src/RooJSONFactoryWSTool.cxx @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -31,13 +32,13 @@ #include "JSONIOUtils.h" #include "HS3Builder.h" -#include "HS3Schema.h" #include "JSONReadRecorder.h" #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #include #include #include +#include /** \class RooJSONFactoryWSTool \ingroup roofit_dev_docs_hs3 @@ -105,12 +107,6 @@ You can always get a list of all the available importers and exporters by callin ROOT.RooFit.JSONIO.printImporters() ROOT.RooFit.JSONIO.printExporters() ~~~ - -Alternatively, you can generate a LaTeX version of the available importers and exporters by calling -~~~ {.py} -tool = ROOT.RooJSONFactoryWSTool(ws) -tool.writedoc("hs3.tex") -~~~ */ constexpr auto hs3VersionTag = "0.3"; @@ -146,11 +142,16 @@ struct RooFit::HS3::ToolImportState { } HS3Document const &document; + /// The components of combined distributions that have data, and are read as such. + std::set componentsWithData = HS3::componentsWithData(document); std::unordered_map functions; std::unordered_map distributions; HS3::ParameterPoint const *defaultValues = nullptr; std::unordered_map defaultValuesByName; HS3::DomainRanges ranges{document.domains}; + /// The components of array distributions created in this import, by name, with their canonical form (see + /// arrayComponentForm()). + std::map arrayComponentForms; }; /// The document that the tool is exporting. @@ -172,6 +173,9 @@ struct RooFit::HS3::ToolExportState { /// The constraint terms of the exported HistFactory channels, as written, and the data sets written for them. std::map> constraintTermsOfPdf; std::set writtenConstraintData; + /// The labels that the constraint terms that are written as components of a combined distribution add to its + /// index category, by category. See exportSingleModelConfig(). + std::map> auxiliaryLabels; /// The observable of a constraint term, by the name of the term, see setConstraintObservable(). std::map constraintObservables; /// Set by the exporter of an object that is interpreted by the document. @@ -179,6 +183,22 @@ struct RooFit::HS3::ToolExportState { std::vector channelConstraints; }; +std::string const *RooFit::HS3::importedArrayComponentForm(RooJSONFactoryWSTool &tool, std::string const &name) +{ + if (!tool._import) { + return nullptr; + } + auto found = tool._import->arrayComponentForms.find(name); + return found == tool._import->arrayComponentForms.end() ? nullptr : &found->second; +} + +void RooFit::HS3::recordImportedArrayComponent(RooJSONFactoryWSTool &tool, std::string const &name, std::string form) +{ + if (tool._import) { + tool._import->arrayComponentForms[name] = std::move(form); + } +} + void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist dist, std::vector channelConstraints) { @@ -188,6 +208,34 @@ void RooFit::HS3::setExportedObject(RooJSONFactoryWSTool &tool, HistFactoryDist namespace { +// Append a name to a list that has every name once. +void addUnique(std::vector &names, std::string const &name) +{ + if (std::find(names.begin(), names.end(), name) == names.end()) { + names.push_back(name); + } +} + +template +std::vector namesOf(Coll_t const &objects) +{ + std::vector names; + for (auto const *object : objects) { + names.push_back(object->GetName()); + } + return names; +} + +// Whether the states of a category have these labels, in any order. +bool hasLabels(RooCategory const &cat, std::vector const &labels) +{ + std::set states; + for (auto const &state : cat) { + states.insert(state.first); + } + return states == std::set(labels.begin(), labels.end()); +} + HS3::Binning toBinning(RooAbsBinning const &binning) { if (binning.isUniform()) { @@ -338,11 +386,7 @@ void createVariables(RooWorkspace &ws, std::vector const &domains, RooJSONFactoryWSTool::error("the axis '" + axis->name + "' is categorical, but '" + axis->name + "' already exists and is not a category"); } - std::set existingLabels; - for (auto const &state : *cat) { - existingLabels.insert(state.first); - } - if (existingLabels != std::set(axis->categories.begin(), axis->categories.end())) { + if (!hasLabels(*cat, axis->categories)) { RooJSONFactoryWSTool::error("the categories of the axis '" + axis->name + "' differ from the states of the existing category '" + axis->name + "'"); } @@ -370,6 +414,14 @@ void createVariables(RooWorkspace &ws, std::vector const &domains, namespace { +// Ends an import or an export: its state must not survive it, not even if there is an exception. The state of an +// import refers to the document. +template +struct StateScope { + std::unique_ptr &state; + ~StateScope() { state.reset(); } +}; + // Keeps track of the objects that are being imported. Objects that depend on each other would otherwise recurse until // the stack overflows. class ImportGuard { @@ -393,12 +445,14 @@ class ImportGuard { } // namespace -void RooFit::HS3::report(Diagnostics const &diagnostics) +void RooFit::HS3::report(Diagnostics const &diagnostics, std::string const &object) { std::string errors; for (Diagnostic const &diagnostic : diagnostics) { - const std::string text = - diagnostic.path.empty() ? diagnostic.message : diagnostic.path + ": " + diagnostic.message; + // A path starts with the name of a component, or with the index of an entry if the object is an array. + const bool needsDot = !object.empty() && !diagnostic.path.empty() && diagnostic.path.front() != '['; + const std::string path = object + (needsDot ? "." : "") + diagnostic.path; + const std::string text = path.empty() ? diagnostic.message : path + ": " + diagnostic.message; if (diagnostic.severity == Diagnostic::Severity::Error) { errors += (errors.empty() ? "" : "\n") + text; } else { @@ -449,21 +503,6 @@ std::string genPrefix(const JSONNode &p, bool trailing_underscore) return prefix; } -// helpers for serializing / deserializing binned datasets -void genIndicesHelper(std::vector> &combinations, std::vector &curr_comb, - const std::vector &vars_numbins, size_t curridx) -{ - if (curridx == vars_numbins.size()) { - // we have filled a combination. Copy it. - combinations.emplace_back(curr_comb); - } else { - for (int i = 0; i < vars_numbins[curridx]; ++i) { - curr_comb[curridx] = i; - ::genIndicesHelper(combinations, curr_comb, vars_numbins, curridx + 1); - } - } -} - // Import string attributes ("dict") and boolean tags ("tags") onto `arg`. void importAttributes(RooAbsArg *arg, HS3::Attributes const &attributes) { @@ -475,75 +514,46 @@ void importAttributes(RooAbsArg *arg, HS3::Attributes const &attributes) } } -void addIfPresent(RooArgSet &out, RooArgSet const *args) -{ - if (args) { - out.add(*args, true); - } -} - -void collectParameterStepWidthCandidatesFromModelConfigs(RooWorkspace const &workspace, RooArgSet &candidates, - RooArgSet &excluded) +// The step widths of the minimization are the errors of the parameters of the ModelConfigs, and of the parameters of +// the top-level pdfs given the data sets. The observables are no parameters. +void exportParameterStepWidths(RooWorkspace const &workspace, std::vector const &pdfs, + std::vector const &data, std::vector &out) { + RooArgSet candidates; + RooArgSet excluded; + auto addIfPresent = [](RooArgSet &set, RooArgSet const *args) { + if (args) { + set.add(*args, true); + } + }; for (TObject *obj : workspace.allGenericObjects()) { - auto const *mc = dynamic_cast(obj); - if (!mc) { - continue; + if (auto const *mc = dynamic_cast(obj)) { + addIfPresent(candidates, mc->GetParametersOfInterest()); + addIfPresent(candidates, mc->GetNuisanceParameters()); + addIfPresent(excluded, mc->GetObservables()); + addIfPresent(excluded, mc->GetGlobalObservables()); + addIfPresent(excluded, mc->GetConditionalObservables()); } - - addIfPresent(candidates, mc->GetParametersOfInterest()); - addIfPresent(candidates, mc->GetNuisanceParameters()); - - addIfPresent(excluded, mc->GetObservables()); - addIfPresent(excluded, mc->GetGlobalObservables()); - addIfPresent(excluded, mc->GetConditionalObservables()); } -} - -void collectParameterStepWidthCandidatesFromPdfs(std::vector const &pdfs, - std::vector const &data, RooArgSet &candidates, - RooArgSet &excluded) -{ for (RooAbsPdf const *pdf : pdfs) { RooArgSet observables; for (RooAbsData const *dataset : data) { std::unique_ptr pdfObs{pdf->getObservables(*dataset->get())}; observables.add(*pdfObs, true); } - - if (observables.empty()) { - continue; + if (!observables.empty()) { + RooArgSet params; + pdf->getParameters(&observables, params); + candidates.add(params, true); + excluded.add(observables, true); } - - RooArgSet params; - pdf->getParameters(&observables, params); - candidates.add(params, true); - excluded.add(observables, true); } -} - -void exportParameterStepWidths(RooWorkspace const &workspace, std::vector const &pdfs, - std::vector const &data, std::vector &out) -{ - RooArgSet candidates; - RooArgSet excluded; - - collectParameterStepWidthCandidatesFromModelConfigs(workspace, candidates, excluded); - collectParameterStepWidthCandidatesFromPdfs(pdfs, data, candidates, excluded); - candidates.sort(); - for (RooAbsArg *arg : candidates) { - if (excluded.find(*arg)) { - continue; - } - auto *var = dynamic_cast(arg); - if (!var || !var->hasError()) { - continue; + if (var && var->hasError() && !excluded.find(*arg)) { + out.push_back({var->GetName(), var->getError()}); } - - out.push_back({var->GetName(), var->getError()}); } } @@ -561,360 +571,66 @@ void importParameterStepWidths(RooWorkspace &workspace, HS3Document const &doc) } } -// Generate all combinations of bin indices for the RooRealVars in `vars`. -std::vector> generateBinIndices(const RooArgSet &vars) -{ - std::vector> combinations; - std::vector vars_numbins; - vars_numbins.reserve(vars.size()); - for (const RooAbsArg *arg : vars) { - // Continuous variables and categories (one bin per state) - vars_numbins.push_back(dynamic_cast(*arg).numBins()); - } - std::vector curr_comb(vars.size()); - ::genIndicesHelper(combinations, curr_comb, vars_numbins, 0); - return combinations; -} - -template -JSONNode const *findRooFitInternal(JSONNode const &node, Keys_t const &...keys) -{ - return node.find("misc", "ROOT_internal", keys...); -} - -// --------------------------------------------------------------------------------------------------------------- -// Distinct variates. HS3 requires the terms of a likelihood, and the components of a combined distribution or data -// set, to have distinct variates: a variate in two of them is one quantity modelled twice (e.g. a constraint that is -// a term of the likelihood and a factor of a channel), or two quantities with one name (e.g. the observable "mass" -// of two channels, which need their own names, "mass_SR" and "mass_CR"). All cases are checked with the same -// function, sharedVariates(), on the document, after the export and before the import. - -bool isName(JSONNode const &node) +// Check that the terms of the likelihoods have distinct variates (see HS3::sharedVariates()) and list the constraints +// of their HistFactory modifiers (see HS3::unlistedConstraints()). Documents of versions before 0.3, which RooFit +// wrote with shared variates itself, are read with a warning. Returns the bookkeeping columns, which may be shared. +std::set checkVariates(HS3Document const &doc) { - return !node.is_container() && !node.is_null() && !node.is_number(); -} - -// Add the names in `node` (one name or a list) to `out`. -void addNames(JSONNode const &node, std::set &out) -{ - if (node.is_seq()) { - for (JSONNode const &child : node.children()) { - addNames(child, out); - } - } else if (isName(node)) { - out.insert(node.val()); - } -} - -// The child `name` of the section `section` of the document, or nullptr. -JSONNode const *findInSection(JSONNode const &rootnode, const char *section, std::string const &name) -{ - JSONNode const *node = rootnode.find(section); - return node ? RooJSONFactoryWSTool::findNamedChild(*node, name) : nullptr; -} - -std::string typeOf(JSONNode const &node) -{ - return node.has_child("type") ? node["type"].val() : ""; -} - -// The names of the components of a combined distribution or data set, written as {"key": ..., : }. -std::vector combinedComponents(JSONNode const &node, const char *field) -{ - std::vector out; - if (JSONNode const *components = node.find("components")) { - for (JSONNode const &component : components->children()) { - if (JSONNode const *name = component.find(field); name && isName(*name)) { - out.push_back(name->val()); - } - } - } - return out; -} - -// The variates of the distribution `name` of the document: the "x" of a fundamental distribution (a name or a list) -// and the axes of a distribution with axes (e.g. histfactory_dist), through products, mixtures and combined -// distributions, and the "x" of the components of an array distribution. The constraint terms named by a -// histfactory_dist are terms of their own, see likelihoodTerms(). -void collectVariates(JSONNode const &rootnode, std::string const &name, std::set &out, - std::set &visited) -{ - if (!visited.insert(name).second) { - return; - } - JSONNode const *node = findInSection(rootnode, "distributions", name); - if (!node) { - return; - } - std::string const type = typeOf(*node); - auto recurse = [&](std::vector const &parts) { - for (std::string const &part : parts) { - collectVariates(rootnode, part, out, visited); - } - }; - auto namesIn = [&](const char *key) { - std::vector parts; - if (JSONNode const *list = node->find(key)) { - for (JSONNode const &part : list->children()) { - if (isName(part)) { - parts.push_back(part.val()); - } - } - } - return parts; - }; - if (type == "product_dist") { - recurse(namesIn("factors")); - } else if (type == "mixture_dist") { - recurse(namesIn("summands")); - } else if (type == "combined_dist") { - recurse(combinedComponents(*node, "dist")); - } else if (type == "array_dist") { - if (JSONNode const *components = node->find("components")) { - for (JSONNode const &component : components->children()) { - if (JSONNode const *x = component.find("x")) { - addNames(*x, out); - } - } - } - } else { - if (JSONNode const *x = node->find("x")) { - addNames(*x, out); - } - if (JSONNode const *axes = node->find("axes")) { - for (JSONNode const &axis : axes->children()) { - if (JSONNode const *axisName = axis.find("name")) { - addNames(*axisName, out); - } - } - } - } -} - -std::set variatesOf(JSONNode const &rootnode, std::string const &name) -{ - std::set variates; - std::set visited; - collectVariates(rootnode, name, variates, visited); - return variates; -} - -// The terms of the likelihood `likelihood` of the document, each once: its distributions (the components of a -// combined distribution, or of a RooSimultaneous written to ROOT_internal by older versions, are terms of their own), -// its "aux_distributions", and the constraint terms named by the modifiers of its histfactory_dist terms. -std::vector likelihoodTerms(JSONNode const &rootnode, JSONNode const &likelihood) -{ - std::vector terms; - auto add = [&](std::string const &term) { - if (std::find(terms.begin(), terms.end(), term) == terms.end()) { - terms.push_back(term); - } - }; - for (const char *key : {"distributions", "aux_distributions"}) { - JSONNode const *list = likelihood.find(key); - if (!list) { - continue; - } - for (JSONNode const &item : list->children()) { - if (!isName(item)) { - continue; - } - std::string const name = item.val(); - if (JSONNode const *node = findInSection(rootnode, "distributions", name)) { - if (typeOf(*node) == "combined_dist") { - for (std::string const &component : combinedComponents(*node, "dist")) { - add(component); - } - continue; - } - } else if (JSONNode const *legacy = findRooFitInternal(rootnode, "combined_distributions", name)) { - if (JSONNode const *components = legacy->find("distributions")) { - for (JSONNode const &component : components->children()) { - add(component.val()); - } - continue; - } - } - add(name); - } - } - for (std::size_t i = 0; i < terms.size(); ++i) { - JSONNode const *node = findInSection(rootnode, "distributions", terms[i]); - if (!node || typeOf(*node) != "histfactory_dist" || !node->has_child("samples")) { - continue; - } - for (JSONNode const &sample : (*node)["samples"].children()) { - if (JSONNode const *modifiers = sample.find("modifiers")) { - for (JSONNode const &modifier : modifiers->children()) { - if (JSONNode const *constraints = modifier.find("constraints")) { - std::set names; - addNames(*constraints, names); - for (std::string const &constraint : names) { - add(constraint); - } - } - } - } - } - } - return terms; -} - -// The parts (terms or components) of something that must have distinct variates, with their variates. -using PartVariates = std::vector>>; - -// The variates that several of `parts` share, one message each. `what` names the parts and `context` the whole, -// e.g. "terms" of "the likelihood 'lh'". -std::vector sharedVariates(PartVariates const &parts, std::string const &what, std::string const &context) -{ - std::map> partsOfVariate; - std::vector order; - for (auto const &[part, variates] : parts) { - for (std::string const &variate : variates) { - auto &list = partsOfVariate[variate]; - if (list.empty()) { - order.push_back(variate); - } - if (std::find(list.begin(), list.end(), part) == list.end()) { - list.push_back(part); - } + const bool isLegacy = doc.metadata.version < HS3::Version{0, 3}; + std::set bookkeepingColumns; + for (auto const &[dataset, columns] : HS3::bookkeepingColumns(doc)) { + for (std::string const &column : columns) { + RooJSONFactoryWSTool::warning( + "RooFitHS3: the axis '" + column + "' of the components of the combined data set '" + dataset + + "' is declared as a bookkeeping column without statistical meaning (misc.ROOT_internal." + "bookkeeping_columns), so it may be shared by the components, although HS3 requires distinct axes."); + bookkeepingColumns.insert(column); } } - std::vector messages; - for (std::string const &variate : order) { - auto const &list = partsOfVariate[variate]; - if (list.size() < 2) { + // The constraints named by HistFactory modifiers, and distinct variates. A document can violate both, e.g. with a + // constraint that is listed twice, so all violations are reported together. + std::vector violations; + std::vector notes; + if (!isLegacy) { + for (std::string const &message : HS3::unlistedConstraints(doc, ¬es)) { + violations.push_back("invalid HS3: " + message + + ". The \"constraints\" of a modifier add no term to the likelihood, which has to list " + "them."); + } + } + for (std::string const &message : HS3::sharedVariates(doc)) { + if (!isLegacy) { + violations.push_back( + "invalid HS3: " + message + + ". The terms of a likelihood and the components of a combined distribution or data set must have distinct " + "variates, apart from identical factors within a combined distribution. Distributions that share factors, " + "e.g. the models of a combination with common constraint terms, are joined in a combined distribution, " + "which the likelihood lists as one term."); continue; } - std::string names; - for (std::size_t i = 0; i < list.size(); ++i) { - names += (i == 0 ? "'" : (i + 1 == list.size() ? "' and '" : "', '")) + list[i]; - } - messages.push_back("the variate '" + variate + "' is shared by the " + what + " " + names + "' of " + context); - } - return messages; -} - -// The axes of the components of a combined data set, by component. -PartVariates combinedDataAxes(JSONNode const &rootnode, JSONNode const &dataset) -{ - PartVariates parts; - for (std::string const &component : combinedComponents(dataset, "dataset")) { - std::set axes; - if (JSONNode const *node = findInSection(rootnode, "data", component)) { - if (JSONNode const *axesNode = node->find("axes")) { - for (JSONNode const &axis : axesNode->children()) { - if (JSONNode const *axisName = axis.find("name")) { - addNames(*axisName, axes); - } - } - } - } - parts.emplace_back(component, axes); - } - return parts; -} - -// Bookkeeping columns (a RooFit extension, in misc.ROOT_internal.bookkeeping_columns as {"": -// [, ...]}): axes that several components of a combined data set share, but that are no variate of any -// distribution, e.g. an event number. They have no statistical meaning, and are exempt from the requirement of -// distinct axes. A declared column that is a variate of a distribution is not a bookkeeping column. -std::map> bookkeepingColumnsOfDocument(JSONNode const &rootnode) -{ - std::map> out; - JSONNode const *declared = findRooFitInternal(rootnode, "bookkeeping_columns"); - if (!declared || !declared->is_map()) { - return out; + RooJSONFactoryWSTool::warning("RooFitHS3: " + message + + ". HS3 0.3 requires distinct variates; the document is read anyway, since it " + "declares an older version."); } - std::set variates; - if (JSONNode const *distributions = rootnode.find("distributions")) { - for (JSONNode const &dist : distributions->children()) { - std::set const v = variatesOf(rootnode, RooJSONFactoryWSTool::name(dist)); - variates.insert(v.begin(), v.end()); + if (!violations.empty()) { + std::string all; + for (std::string const &violation : violations) { + all += (all.empty() ? "" : "\n") + violation; } + RooJSONFactoryWSTool::error(all); } - for (JSONNode const &entry : declared->children()) { - for (JSONNode const &column : entry.children()) { - if (!variates.count(column.val())) { - out[entry.key()].insert(column.val()); - } - } + for (std::string const ¬e : notes) { + RooJSONFactoryWSTool::warning("RooFitHS3: " + note + "."); } - return out; + return bookkeepingColumns; } -// All shared variates of the document: of the terms of each likelihood, of the components of the combined -// distributions that no likelihood lists (the others are checked through the likelihood), and the shared axes of the -// components of all combined data sets, except for bookkeeping columns. -std::vector sharedVariatesOfDocument(JSONNode const &rootnode) +// Declare the bookkeeping columns of the exported document (see HS3::bookkeepingColumns()): the axes that several +// components of a combined data set share, and whose variable in the workspace `ws` is used by nothing else, i.e. has +// no clients and is in no set of a ModelConfig. RooAbsData::split() keeps such a column (e.g. an event number) in +// every sub-data set. +void declareBookkeepingColumns(HS3Document &doc, RooWorkspace const &ws) { - std::vector messages; - auto append = [&](std::vector const &more) { - messages.insert(messages.end(), more.begin(), more.end()); - }; - std::set inLikelihoods; - if (JSONNode const *likelihoods = rootnode.find("likelihoods")) { - for (JSONNode const &likelihood : likelihoods->children()) { - for (const char *key : {"distributions", "data", "aux_distributions"}) { - if (JSONNode const *list = likelihood.find(key)) { - for (JSONNode const &item : list->children()) { - inLikelihoods.insert(item.val()); - } - } - } - PartVariates parts; - for (std::string const &term : likelihoodTerms(rootnode, likelihood)) { - parts.emplace_back(term, variatesOf(rootnode, term)); - } - append(sharedVariates(parts, "terms", "the likelihood '" + RooJSONFactoryWSTool::name(likelihood) + "'")); - } - } - if (JSONNode const *distributions = rootnode.find("distributions")) { - for (JSONNode const &dist : distributions->children()) { - std::string const name = RooJSONFactoryWSTool::name(dist); - if (typeOf(dist) != "combined_dist" || inLikelihoods.count(name)) { - continue; - } - PartVariates parts; - for (std::string const &component : combinedComponents(dist, "dist")) { - parts.emplace_back(component, variatesOf(rootnode, component)); - } - append(sharedVariates(parts, "components", "the combined_dist '" + name + "'")); - } - } - // The axes of the components of every combined data set, also of those that a likelihood lists, except for the - // bookkeeping columns. - std::map> const bookkeeping = bookkeepingColumnsOfDocument(rootnode); - if (JSONNode const *data = rootnode.find("data")) { - for (JSONNode const &dataset : data->children()) { - std::string const name = RooJSONFactoryWSTool::name(dataset); - if (typeOf(dataset) != "combined") { - continue; - } - PartVariates parts = combinedDataAxes(rootnode, dataset); - if (auto found = bookkeeping.find(name); found != bookkeeping.end()) { - for (auto &part : parts) { - for (std::string const &column : found->second) { - part.second.erase(column); - } - } - } - append(sharedVariates(parts, "components", "the combined data set '" + name + "'")); - } - } - return messages; -} - -// Declare the bookkeeping columns of the exported document `rootnode` (see bookkeepingColumnsOfDocument()): the axes -// that several components of a combined data set share, and whose variable in the workspace `ws` is used by nothing -// else, i.e. has no clients and is in no set of a ModelConfig. RooAbsData::split() keeps such a column (e.g. an event -// number) in every sub-data set. -void declareBookkeepingColumns(JSONNode &rootnode, RooWorkspace const &ws) -{ - JSONNode const *data = rootnode.find("data"); - if (!data) { - return; - } std::vector modelConfigSets; for (TObject *obj : ws.allGenericObjects()) { if (auto *mc = dynamic_cast(obj)) { @@ -934,118 +650,15 @@ void declareBookkeepingColumns(JSONNode &rootnode, RooWorkspace const &ws) return std::none_of(modelConfigSets.begin(), modelConfigSets.end(), [&](RooArgSet const *set) { return set->find(name.c_str()) != nullptr; }); }; - std::map> columns; - for (JSONNode const &dataset : data->children()) { - if (typeOf(dataset) != "combined") { - continue; - } - std::map counts; - for (auto const &[component, axes] : combinedDataAxes(rootnode, dataset)) { - for (std::string const &axis : axes) { - ++counts[axis]; - } - } - for (auto const &[axis, count] : counts) { - if (count > 1 && usedByNothing(axis)) { - columns[RooJSONFactoryWSTool::name(dataset)].insert(axis); + for (auto const &[dataset, axes] : HS3::sharedAxes(doc)) { + for (std::string const &axis : axes) { + if (usedByNothing(axis)) { + doc.rooFit.bookkeepingColumns[dataset].push_back(axis); } } } - if (columns.empty()) { - return; - } - JSONNode &node = rootnode["misc"]["ROOT_internal"]["bookkeeping_columns"].set_map(); - for (auto const &[dataset, names] : columns) { - node[dataset].fill_seq(names); - } } -// Write each term of the likelihoods of the exported document `rootnode` once. A RooFit model often has a constraint -// both as a factor of the product of a channel and as a term of the likelihood (e.g. a constraint of a HistFactory -// modifier, an external constraint of the ModelConfig, or any constraint of a channel, see -// constraintsOfChannels()); RooFit counts it once. In HS3 it would enter twice, so the factor is removed from -// the product. Only products that no other distribution uses are changed (being a component -// of a combined distribution is not a use), and only factors that are terms of every likelihood that has the product -// as a term. -void writeLikelihoodTermsOnce(JSONNode &rootnode) -{ - JSONNode const *likelihoods = rootnode.find("likelihoods"); - JSONNode const *distributions = rootnode.find("distributions"); - if (!likelihoods || !distributions) { - return; - } - // The products that are terms, with the factors that are terms wherever the product is one. - std::map> removable; - for (JSONNode const &likelihood : likelihoods->children()) { - std::vector const terms = likelihoodTerms(rootnode, likelihood); - for (std::string const &term : terms) { - JSONNode const *node = findInSection(rootnode, "distributions", term); - if (!node || typeOf(*node) != "product_dist" || !node->has_child("factors")) { - continue; - } - std::set factorsThatAreTerms; - for (JSONNode const &factor : (*node)["factors"].children()) { - if (factor.val() != term && std::find(terms.begin(), terms.end(), factor.val()) != terms.end()) { - factorsThatAreTerms.insert(factor.val()); - } - } - auto found = removable.find(term); - if (found == removable.end()) { - removable[term] = factorsThatAreTerms; - } else { - std::set both; - std::set_intersection(found->second.begin(), found->second.end(), factorsThatAreTerms.begin(), - factorsThatAreTerms.end(), std::inserter(both, both.begin())); - found->second = both; - } - } - } - // Products that another distribution uses are left alone. - for (JSONNode const &dist : distributions->children()) { - std::vector used; - for (const char *key : {"factors", "summands"}) { - if (JSONNode const *parts = dist.find(key)) { - for (JSONNode const &part : parts->children()) { - if (isName(part)) { - used.push_back(part.val()); - } - } - } - } - if (typeOf(dist) != "combined_dist") { - for (std::string const &component : combinedComponents(dist, "dist")) { - used.push_back(component); - } - } - for (std::string const &name : used) { - removable.erase(name); - } - } - for (auto const &[product, remove] : removable) { - if (remove.empty()) { - continue; - } - auto &node = const_cast(*findInSection(rootnode, "distributions", product)); - std::vector keep; - for (JSONNode const &factor : node["factors"].children()) { - if (remove.find(factor.val()) == remove.end()) { - keep.push_back(factor.val()); - } - } - if (keep.empty()) { - continue; - } - for (std::string const &factor : remove) { - oocoutI(nullptr, IO) << "RooFitHS3: '" << factor << "' is a factor of '" << product - << "' and a term of its likelihood; it is written once, as a term of the likelihood" - << std::endl; - } - node["factors"].clear(); - node["factors"].fill_seq(keep); - } -} -// --------------------------------------------------------------------------------------------------------------- - // Export the string attributes and tags of `arg` into the ROOT-internal attributes. void exportAttributes(const RooAbsArg *arg, std::map &attributes) { @@ -1061,7 +674,6 @@ void exportAttributes(const RooAbsArg *arg, std::mapstringAttributes()) { // Skip some RooFit internals if (it.first == "factory_tag" || it.first == "PROD_TERM_TYPE") @@ -1071,7 +683,8 @@ void exportAttributes(const RooAbsArg *arg, std::mapattributes()) { // Skip some RooFit internals. The array distribution tag is restored from the type on import. - if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object" || attr == arrayDistTag) + if (attr == "SnapShot_ExtRefClone" || attr == "RooRealConstant_Factory_Object" || attr == arrayDistTag || + attr == helperObjectTag) continue; initialize(); out->tags.push_back(attr); @@ -1083,13 +696,12 @@ void exportAttributes(const RooAbsArg *arg, std::map createObservable(HS3::BinnedAxis const &axis) +// The axis of unbinned or point data for a variable. Infinite bounds are left out: the axis is unbounded there. +HS3::ContinuousAxis toContinuousAxis(RooRealVar const &var) { - auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); - applyBinning(*obs, axis.binning); - return obs; + RooJSONFactoryWSTool::testValidName(var.GetName(), false); + HS3::ContinuousAxis axis; + axis.name = var.GetName(); + if (!RooNumber::isInfinite(var.getMin())) { + axis.min = var.getMin(); + } + if (!RooNumber::isInfinite(var.getMax())) { + axis.max = var.getMax(); + } + return axis; } -// The labels of a category, as a set. -std::set labelsOf(RooCategory const &cat) +// A categorical axis has the labels in the order of the states of the category, which is the order of the bins of a +// RooDataHist. +HS3::CategoricalAxis toCategoricalAxis(RooAbsCategory const &cat) { - std::set labels; - for (auto const &state : cat) { - labels.insert(state.first); + HS3::CategoricalAxis axis; + axis.name = cat.GetName(); + for (unsigned int i = 0; i < cat.size(); ++i) { + axis.categories.push_back(cat.getOrdinal(i).first); } - return labels; + return axis; +} + +// Binned data over variables that are RooRealVars or RooCategories. +HS3::BinnedData toBinnedData(RooArgSet const &vars, std::size_t n, double const *contents) +{ + HS3::BinnedData data; + for (RooAbsArg const *var : vars) { + if (auto const *cat = dynamic_cast(var)) { + RooJSONFactoryWSTool::testValidName(cat->GetName(), false); + data.axes.emplace_back(toCategoricalAxis(*cat)); + } else { + data.axes.emplace_back(HS3::toBinnedAxis(static_cast(*var))); + } + } + data.contents.assign(contents, contents + n); + return data; +} + +std::unique_ptr createObservable(HS3::BinnedAxis const &axis) +{ + auto obs = std::make_unique(axis.name.c_str(), axis.name.c_str(), axis.min, axis.max); + applyBinning(*obs, axis.binning); + return obs; } // Create the variables for the axes of binned data. A categorical axis is one bin per category. HS3 identifies @@ -1143,7 +788,7 @@ RooArgSet createBinnedObservables(std::vector(axisVariant); auto cat = std::make_unique(axis.name.c_str(), axis.name.c_str()); if (RooCategory const *wsCat = workspace.cat(axis.name)) { - if (labelsOf(*wsCat) != std::set(axis.categories.begin(), axis.categories.end())) { + if (!hasLabels(*wsCat, axis.categories)) { RooJSONFactoryWSTool::error("the categories of the axis '" + axis.name + "' of the dataset '" + dataName + "' differ from those of the category '" + axis.name + "'"); } @@ -1208,16 +853,15 @@ std::vector permute(std::vector const &values, std::vectorcontains(axis.name)) { - var.setRange(*axis.min, *axis.max); - } else if (var.getMin() != *axis.min || var.getMax() != *axis.max) { + var.setRange(min, max); + } else if (var.getMin() != min || var.getMax() != max) { std::stringstream ss; - ss << "RooFitHS3: the range [" << *axis.min << ", " << *axis.max << "] of the axis '" << axis.name - << "' of the dataset '" << dataName << "' differs from the range [" << var.getMin() << ", " << var.getMax() - << "] of the variable."; + ss << "RooFitHS3: the range [" << min << ", " << max << "] of the axis '" << axis.name << "' of the dataset '" + << dataName << "' differs from the range [" << var.getMin() << ", " << var.getMax() << "] of the variable."; RooJSONFactoryWSTool::warning(ss.str()); } } @@ -1237,7 +881,7 @@ std::unique_ptr createUnbinnedData(std::string const &name, HS3::Unb "\" could not be found in the workspace! Categorical observables are " "declared by a categorical axis in a domain."); } - if (labelsOf(*cat) != std::set(catAxis.categories.begin(), catAxis.categories.end())) { + if (!hasLabels(*cat, catAxis.categories)) { RooJSONFactoryWSTool::error("The categories of the axis \"" + catAxis.name + "\" differ from the states of the category \"" + catAxis.name + "\""); } @@ -1296,7 +940,25 @@ std::unique_ptr loadData(HS3::Data const &input, RooWorkspace &works } return createUnbinnedData(input.name, *unbinned, workspace, ranges); } - RooJSONFactoryWSTool::error("datasets of type \"point\" like '" + input.name + "' are not supported yet"); + // Point data is the observation of a single variable. With its axis, it is the same as unbinned data with a single + // entry, and becomes a RooDataSet with one entry. + auto const &point = std::get(input.body); + if (point.uncertainty) { + RooJSONFactoryWSTool::warning("RooFitHS3: the \"uncertainty\" of the dataset '" + input.name + + "' is ignored: HS3 doesn't define how it enters a likelihood."); + } + // Without an axis, the value is the one of the single variate of the paired distribution, see + // HS3::resolvePointData(). + if (point.axes.empty()) { + RooJSONFactoryWSTool::error("the point data set '" + input.name + + "' has no axis, and it is not paired with a distribution that has a single variate"); + } else if (point.axes.size() != 1) { + RooJSONFactoryWSTool::error("the point data set '" + input.name + "' must have exactly one axis"); + } + HS3::UnbinnedData unbinned; + unbinned.axes.emplace_back(point.axes.front()); + unbinned.entries.push_back(point.value); + return createUnbinnedData(input.name, unbinned, workspace, ranges); } // The version of ROOT that writes the document, with dots only such that it can be read automatically. @@ -1307,6 +969,14 @@ std::string rootVersion() return versionName; } +// The data set `name` among the loaded ones that are not a component of a combined data set by now, or the data of a +// constraint term, or nullptr. +std::unique_ptr *findDataset(std::vector> &datasets, std::string const &name) +{ + auto found = std::find_if(datasets.begin(), datasets.end(), [&](auto &d) { return d && d->GetName() == name; }); + return found != datasets.end() ? &*found : nullptr; +} + // A distribution of the document can be a function in RooFit, or it can be skipped. void requirePdf(RooAbsArg const *arg, std::string const &what) { @@ -1324,10 +994,7 @@ void requirePdf(RooAbsArg const *arg, std::string const &what) void checkCombinedPairing(RooSimultaneous const &simPdf, std::string const &dataName, HS3::CombinedData const &data) { std::vector columns = selectionColumns(simPdf.indexCat()); - std::vector columnNames; - for (RooCategory const *column : columns) { - columnNames.push_back(column->GetName()); - } + const std::vector columnNames = namesOf(columns); std::vector dataColumns = data.columns; std::sort(dataColumns.begin(), dataColumns.end()); if (dataColumns != columnNames) { @@ -1336,7 +1003,7 @@ void checkCombinedPairing(RooSimultaneous const &simPdf, std::string const &data } // The keys of the data, each as a map from the column to the label. std::set> dataKeys; - for (HS3::CombinedDataComponent const &component : data.components) { + for (auto const &component : data.components) { std::map key; for (std::size_t i = 0; i < data.columns.size(); ++i) { key[data.columns[i]] = component.key[i]; @@ -1356,95 +1023,589 @@ void checkCombinedPairing(RooSimultaneous const &simPdf, std::string const &data if (!component) { continue; } - std::map key; - for (std::size_t i = 0; i < columns.size(); ++i) { - key[columnNames[i]] = labels[i]; + std::map key; + for (std::size_t i = 0; i < columns.size(); ++i) { + key[columnNames[i]] = labels[i]; + } + pdfKeys.insert(key); + if (dataKeys.count(key) > 0) { + continue; + } + if (component->canBeExtended()) { + RooJSONFactoryWSTool::error( + "the combined data set '" + dataName + "' has no sub-data set for the key " + keyToString(key) + + " of the extended component '" + component->GetName() + "' of '" + simPdf.GetName() + + "'. RooFit cannot leave a component unobserved: it would add the expected number of events of the " + + "component to the likelihood, see https://github.com/root-project/root/issues/24004"); + } + RooJSONFactoryWSTool::warning("RooFitHS3: the combined data set '" + dataName + + "' has no sub-data set for the key " + keyToString(key) + " of the component '" + + component->GetName() + "' of '" + simPdf.GetName() + + "', which RooFit treats as present and empty."); + } + for (auto const &key : dataKeys) { + if (pdfKeys.count(key) == 0) { + RooJSONFactoryWSTool::warning("RooFitHS3: the sub-data set for the key " + keyToString(key) + + " of the combined data set '" + dataName + "' is ignored, because '" + + simPdf.GetName() + "' has no component for it."); + } + } +} + +// The observables of the analysis `analysis` if no data set of its likelihood gives them: the list that RooFit writes +// in misc.ROOT_internal.ModelConfigs for a ModelConfig without data, or else the variables that HS3 makes +// observables. By the standard, all parameters of the distributions of the likelihood are listed in a domain of the +// analysis or are constant, so a variate of the distributions of the likelihood (see HS3::variatesOf(), the columns +// of a combined distribution are observables as well) that is neither, is an observable. Requiring all three excludes +// the parameters, and the observed values of the constraint terms, which are constant parameters. +void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysis, HS3::Likelihood const &likelihood, + HS3::ModelConfigInfo const *mcInfo, RooWorkspace &workspace, RooArgSet &observables) +{ + if (mcInfo && mcInfo->observables) { + for (std::string const &name : *mcInfo->observables) { + if (RooAbsArg *arg = workspace.arg(name)) { + observables.add(*arg); + } else { + RooJSONFactoryWSTool::warning("RooFitHS3: the observable '" + name + "' of the analysis '" + analysis.name + + "' (misc.ROOT_internal.ModelConfigs) does not exist"); + } + } + return; + } + std::set inDomains; + for (auto const &domainRef : analysis.domains) { + if (HS3::Domain const *domain = HS3::findByName(doc.domains, domainRef.name)) { + for (auto const &axis : domain->axes) { + inDomains.insert(HS3::nameOf(axis)); + } + } + } + for (auto const &poi : analysis.parametersOfInterest) { + inDomains.insert(poi.name); + } + std::string list; + for (std::string const &variate : HS3::variatesOf(doc, likelihood)) { + RooAbsArg *arg = workspace.arg(variate); + if (!dynamic_cast(arg) && !dynamic_cast(arg)) { + continue; + } + if (inDomains.count(variate) || arg->isConstant()) { + continue; + } + observables.add(*arg); + list += (list.empty() ? "'" : ", '") + variate + "'"; + } + if (!list.empty()) { + oocoutI(nullptr, IO) << "RooFitHS3: no data set of the likelihood of the analysis '" << analysis.name + << "' gives its observables; they are the variates of the likelihood that are neither in a " + "domain of the analysis nor constant: " + << list << std::endl; + } +} + +// Whether the term `term` of a likelihood is a constraint term of the channel `channel` of a model with the channels +// `channels`: it has a global observable (a variable that no channel depends on) and another variable that the +// channel depends on. An array of constraints is one term. +bool isConstraintOfChannel(RooAbsPdf const &term, RooAbsPdf const &channel, RooArgSet const &channels) +{ + if (&term == &channel || channels.containsInstance(term)) { + return false; + } + RooArgSet leaves; + term.leafNodeServerList(&leaves); + RooArgSet parameters; + bool hasGlobalObservable = false; + for (RooAbsArg *leaf : leaves) { + auto *var = dynamic_cast(leaf); + if (!var) { + continue; + } + // Whether a global observable is constant yet depends on where the document has its value: as a parameter, or + // in the data set of the constraint, which is read later. + if (std::none_of(channels.begin(), channels.end(), + [&](RooAbsArg const *other) { return other->dependsOn(*var); })) { + hasGlobalObservable = true; + } else { + parameters.add(*var); + } + } + return hasGlobalObservable && !parameters.empty() && channel.dependsOn(parameters); +} + +// The other terms of the likelihoods of the document that have the distribution `dist` as a term: their other +// distributions that can be constraint terms, and their auxiliary distributions. These are the candidates for +// constraint terms of `dist`. +std::vector otherLikelihoodTerms(HS3Document const &doc, std::string const &dist) +{ + std::vector out; + auto add = [&](std::string const &name) { + if (name != dist) { + addUnique(out, name); + } + }; + // A term can be a constraint term of the channels if it is a measurement of one observation: paired with nothing, + // a value given in-line, point data or an unbinned data set of one entry. A combined distribution, or a term paired + // with a data set of several entries (e.g. the model of another document that the likelihood combines with this + // one), is a measurement of its own. + auto canBeConstraint = [&](HS3::LikelihoodTerm const &term) { + if (HS3::find(doc.distributions, term.distribution.name)) { + return false; + } + auto const *dataRef = std::get_if(&term.data); + if (!dataRef) { + return true; + } + auto data = + std::find_if(doc.data.begin(), doc.data.end(), [&](auto const &d) { return d.name == dataRef->name; }); + if (data == doc.data.end() || std::holds_alternative(data->body)) { + return true; + } + auto const *unbinned = std::get_if(&data->body); + return unbinned && unbinned->numEntries() == 1; + }; + for (auto const &likelihood : doc.likelihoods) { + if (std::none_of(likelihood.terms.begin(), likelihood.terms.end(), + [&](auto const &term) { return term.distribution.name == dist; })) { + continue; + } + for (auto const &term : likelihood.terms) { + if (canBeConstraint(term)) { + add(term.distribution.name); + } + } + for (auto const &aux : likelihood.auxDistributions) { + add(aux.name); + } + } + return out; +} + +// In HS3, the constraint terms of a model are terms of the likelihood; in RooFit, as HistFactory builds its models, a +// constraint is a factor of every channel whose parameters it constrains, which makes the pdf of the model the +// sampling distribution of the auxiliary measurements as well. The export takes the constraints out of the channels +// (writeLikelihoodTermsOnce()); this puts them back: a constraint term with a global observable (a constant variable +// that the model itself does not depend on) becomes a factor of every channel that is a product and depends on one +// of its other parameters. An array of constraints goes to its channels as a whole. The constraints that +// found a channel are removed from `constraints`; the others stay external constraints of the ModelConfig, e.g. a +// penalty term without an observation, like the regularization of an unfolding, or a constraint of a model whose +// channels are not products. +void attachConstraintsToChannels(RooAbsPdf &pdf, RooArgSet &constraints) +{ + RooArgSet channels; + if (auto *simPdf = dynamic_cast(&pdf)) { + for (auto const &state : simPdf->indexCat()) { + if (RooAbsPdf *channel = simPdf->getPdf(state.first)) { + channels.add(*channel, /*silent=*/true); + } + } + } else { + channels.add(pdf); + } + RooArgSet attached; + for (auto *constraint : static_range_cast(constraints)) { + // Decided for all channels before any is changed: a channel that has the constraint depends on its global + // observable, which then would not look like one anymore. + bool isFactor = false; + std::vector products; + for (auto *channel : static_range_cast(channels)) { + auto *product = dynamic_cast(channel); + if (!product || channel == constraint) { + continue; + } + if (product->pdfList().find(*constraint)) { + isFactor = true; + } else if (isConstraintOfChannel(*constraint, *channel, channels)) { + products.push_back(product); + } + } + for (RooProdPdf *product : products) { + product->addPdfs(RooArgSet{*constraint}); + } + if (isFactor || !products.empty()) { + attached.add(*constraint); + } + } + constraints.remove(attached); +} + +// RooFit has one pdf per ModelConfig: the terms of a likelihood become one RooSimultaneous `_simPdf`. Its +// channels are the components of the terms that are combined distributions, flattened also where a component is a +// combined distribution itself (HS3 #161), and the other terms. Its index has the columns of all combined +// distributions, with the label "" in a column that a channel doesn't have (HS3 allows relabelling), and the column +// `_categoryIndex`, labelled with the names of the terms, if there is a term that is not a combined +// distribution or if the channels can't be told apart without it. Several columns are a RooSuperCategory. +// +// Within a combined distribution, identical factors enter once (union composition). RooFit finds a constraint term +// that several channels contain once, but array distributions that are channels of their own would each be evaluated +// on their data: the array channels of one component type are merged into one array distribution with the union of +// their components, and their data into one entry, if each has a data set of one entry (or no data at all, in a pure +// model). The terms of a likelihood are a plain product (HS3): the variates of different terms are distinct, which +// the import has checked, so that the flattening adds no union across terms. +// +// The data sets paired with the terms become one data set `_data` with these columns, which replaces them +// unless another likelihood or analysis uses them. The columns are added to `observables`. The terms that the new +// RooSimultaneous replaces are marked as helpers, which the export only writes if something else uses them. +// +// Returns nullptr if the likelihood has a single term that needs none of this. +RooSimultaneous *flatModelOfLikelihood(HS3Document const &doc, HS3::Analysis const &analysis, RooWorkspace &workspace, + std::vector const &terms, + std::vector> &datasets, RooArgSet &observables) +{ + std::string const &analysisName = analysis.name; + struct Channel { + std::size_t term; + std::map labels; // by column + RooAbsPdf *pdf; + }; + std::vector channels; + std::set innerColumns; + bool plainTerm = false; + bool nested = false; + std::function const &)> collect = + [&](std::size_t term, RooAbsPdf *pdf, std::map const &labels) { + auto *sim = dynamic_cast(pdf); + if (!sim) { + channels.push_back({term, labels, pdf}); + return; + } + nested |= !labels.empty(); + std::vector const columns = selectionColumns(sim->indexCat()); + for (RooCategory const *column : columns) { + innerColumns.insert(column->GetName()); + } + for (std::vector const &key : selectionKeys(columns)) { + if (RooAbsPdf *component = sim->getPdf(selectionLabel(sim->indexCat(), columns, key))) { + std::map inner = labels; + for (std::size_t k = 0; k < columns.size(); ++k) { + if (!inner.emplace(columns[k]->GetName(), key[k]).second) { + RooJSONFactoryWSTool::error("the combined distribution '" + std::string(sim->GetName()) + + "' has the column '" + columns[k]->GetName() + + "', which a combined distribution that contains it has as well"); + } + } + collect(term, component, inner); + } + } + }; + for (std::size_t i = 0; i < terms.size(); ++i) { + RooAbsPdf *pdf = workspace.pdf(terms[i]->distribution.name); + plainTerm |= !dynamic_cast(pdf); + collect(i, pdf, {}); + } + + // The array channels that are merged, by component type. + auto componentType = [](RooAbsPdf const *pdf) -> std::string { + auto const *array = dynamic_cast(pdf); + if (!array || !pdf->getAttribute(arrayDistTag) || array->pdfList().empty()) { + return ""; + } + std::string const type = array->pdfList()[0].ClassName(); + for (RooAbsArg const *component : array->pdfList()) { + if (type != component->ClassName()) { + return ""; + } + } + return type; + }; + std::map> arraysByType; + for (std::size_t c = 0; c < channels.size(); ++c) { + if (std::string const type = componentType(channels[c].pdf); !type.empty()) { + arraysByType[type].push_back(c); + } + } + + // The data: every term must be paired with a data set that has the columns of its channels. + std::vector *> termData; + for (std::size_t i = 0; i < terms.size(); ++i) { + auto const *dataRef = std::get_if(&terms[i]->data); + auto found = dataRef ? std::find_if(datasets.begin(), datasets.end(), + [&](auto const &d) { return d && d->GetName() == dataRef->name; }) + : datasets.end(); + if (found == datasets.end()) { + termData.clear(); + break; + } + bool complete = true; + for (Channel const &channel : channels) { + for (auto const &item : channel.labels) { + complete &= channel.term != i || (*found)->get()->find(item.first.c_str()) != nullptr; + } + } + if (!complete) { + termData.clear(); + break; + } + termData.push_back(&*found); + } + // The entries of the data of each channel, by the labels of the columns of its term. + auto channelOf = [&](std::size_t term, RooArgSet const &values) -> std::size_t { + for (std::size_t c = 0; c < channels.size(); ++c) { + if (channels[c].term != term) { + continue; + } + bool match = true; + for (auto const &[column, label] : channels[c].labels) { + auto const *cat = dynamic_cast(values.find(column.c_str())); + match &= cat && cat->getCurrentLabel() == label; + } + if (match) { + return c; + } + } + return channels.size(); + }; + std::vector entries(channels.size(), 0); + for (std::size_t i = 0; i < termData.size(); ++i) { + RooAbsData &data = **termData[i]; + for (int entry = 0; entry < data.numEntries(); ++entry) { + std::size_t const c = channelOf(i, *data.get(entry)); + if (c < channels.size()) { + ++entries[c]; + } + } + } + // The channel that each array channel is merged into, if any. + std::map mergedInto; + std::map mergedComponents; + for (auto const &[type, members] : arraysByType) { + std::vector candidates; + for (std::size_t c : members) { + if (termData.empty() || entries[c] == 1) { + candidates.push_back(c); + } + } + if (candidates.size() < 2) { + continue; + } + RooArgList &components = mergedComponents[candidates.front()]; + for (std::size_t c : candidates) { + mergedInto[c] = candidates.front(); + // Identical components are one object (see importArrayDist()), which the merged array has once. + for (RooAbsArg *component : static_cast(channels[c].pdf)->pdfList()) { + if (!components.containsInstance(*component)) { + components.add(*component); + } + } } - pdfKeys.insert(key); - if (dataKeys.count(key) > 0) { + } + + if (terms.size() == 1 && !nested && mergedInto.empty()) { + return nullptr; + } + + // The column of the terms is needed if the labels of the other columns don't tell the channels apart. A term that + // is not combined has no labels at all; with the column of the terms, an empty label is never the only one of a + // key, which would be the label of a RooSimultaneous over a single RooCategory. + bool unique = !plainTerm; + std::set> innerKeys; + for (std::size_t c = 0; c < channels.size(); ++c) { + if (mergedInto.count(c) && mergedInto[c] != c) { continue; } - if (component->canBeExtended()) { - RooJSONFactoryWSTool::error( - "the combined data set '" + dataName + "' has no sub-data set for the key " + keyToString(key) + - " of the extended component '" + component->GetName() + "' of '" + simPdf.GetName() + - "'. RooFit cannot leave a component unobserved: it would add the expected number of events of the " + - "component to the likelihood, see https://github.com/root-project/root/issues/24004"); + std::vector key; + for (std::string const &column : innerColumns) { + auto found = channels[c].labels.find(column); + key.push_back(found == channels[c].labels.end() ? "" : found->second); } - RooJSONFactoryWSTool::warning("RooFitHS3: the combined data set '" + dataName + - "' has no sub-data set for the key " + keyToString(key) + " of the component '" + - component->GetName() + "' of '" + simPdf.GetName() + - "', which RooFit treats as present and empty."); + unique = innerKeys.insert(key).second && unique; } - for (auto const &key : dataKeys) { - if (pdfKeys.count(key) == 0) { - RooJSONFactoryWSTool::warning("RooFitHS3: the sub-data set for the key " + keyToString(key) + - " of the combined data set '" + dataName + "' is ignored, because '" + - simPdf.GetName() + "' has no component for it."); + std::string const termColumn = analysisName + "_categoryIndex"; + if (!unique) { + RooCategory outer{termColumn.c_str(), termColumn.c_str()}; + for (std::size_t i = 0; i < terms.size(); ++i) { + outer.defineType(terms[i]->distribution.name, static_cast(i)); } + workspace.import(outer, RooFit::Silence()); } -} -// Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. -// The observables of the analysis `analysis` if no data set of its likelihood gives them: the list that RooFit writes -// in misc.ROOT_internal.ModelConfigs for a ModelConfig without data, or else the variables that HS3 makes -// observables. By the standard, all parameters of the distributions of the likelihood are listed in a domain of the -// analysis or are constant, so a variate of the distributions of the likelihood (`variates`, see collectVariates()) -// that is neither, is an observable. Requiring all three excludes the parameters, and the observed values of the -// constraint terms, which are constant parameters. -void observablesWithoutData(HS3Document const &doc, HS3::Analysis const &analysis, HS3::ModelConfigInfo const *mcInfo, - std::set const &variates, RooWorkspace &workspace, RooArgSet &observables) -{ - if (mcInfo && mcInfo->observables) { - for (std::string const &name : *mcInfo->observables) { - if (RooAbsArg *arg = workspace.arg(name)) { - observables.add(*arg); - } else { - RooJSONFactoryWSTool::warning("RooFitHS3: the observable '" + name + "' of the analysis '" + analysis.name + - "' (misc.ROOT_internal.ModelConfigs) does not exist"); + std::vector columns; + for (std::string const &column : innerColumns) { + columns.push_back(workspace.cat(column)); + } + if (!unique) { + columns.push_back(workspace.cat(termColumn)); + } + std::sort(columns.begin(), columns.end(), + [](RooCategory const *l, RooCategory const *r) { return std::strcmp(l->GetName(), r->GetName()) < 0; }); + std::vector const constColumns(columns.begin(), columns.end()); + + // The key of a channel: the name of its term, its labels, and "" for the columns it doesn't have. + auto keyOf = [&](Channel const &channel) { + std::vector key; + for (RooCategory *column : columns) { + if (column->GetName() == termColumn) { + key.push_back(terms[channel.term]->distribution.name); + continue; + } + auto found = channel.labels.find(column->GetName()); + if (found != channel.labels.end()) { + key.push_back(found->second); + continue; + } + if (!column->hasLabel("")) { + column->defineType(""); } + key.push_back(""); } - return; + return key; + }; + + std::unique_ptr superCat; + RooAbsCategoryLValue *indexCat = columns.front(); + if (columns.size() > 1) { + RooArgSet inputs; + for (RooCategory *column : columns) { + inputs.add(*column); + } + std::string const indexName = analysisName + "_index"; + superCat = std::make_unique(indexName.c_str(), indexName.c_str(), inputs); + indexCat = superCat.get(); } - std::set inDomains; - for (auto const &domainRef : analysis.domains) { - for (HS3::Domain const &domain : doc.domains) { - if (domain.name != domainRef.name) { + std::map pdfMap; + for (std::size_t c = 0; c < channels.size(); ++c) { + RooAbsPdf *pdf = channels[c].pdf; + if (auto found = mergedInto.find(c); found != mergedInto.end()) { + if (found->second != c) { continue; } - for (auto const &axis : domain.axes) { - inDomains.insert(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); - } + // The merged array: the union of the components, one object each. + std::string const name = analysisName + "_" + pdf->GetName() + "_merged"; + RooProdPdf merged{name.c_str(), name.c_str(), mergedComponents[c]}; + merged.setAttribute(arrayDistTag); + workspace.import(merged, RooFit::RecycleConflictNodes(true), RooFit::Silence(true)); + pdf = workspace.pdf(name); } + pdfMap[selectionLabel(*indexCat, constColumns, keyOf(channels[c]))] = pdf; } - for (auto const &poi : analysis.parametersOfInterest) { - inDomains.insert(poi.name); + std::string const simPdfName = analysisName + "_simPdf"; + RooSimultaneous simPdf{simPdfName.c_str(), simPdfName.c_str(), pdfMap, *indexCat}; + workspace.import(simPdf, RooFit::RecycleConflictNodes(true), RooFit::Silence(true)); + for (RooCategory *column : columns) { + observables.add(*column, /*silent=*/true); } - std::string list; - for (std::string const &variate : variates) { - RooAbsArg *arg = workspace.arg(variate); - if (!dynamic_cast(arg) && !dynamic_cast(arg)) { - continue; + // The terms that the RooSimultaneous replaces are written only if something else uses them. + for (HS3::LikelihoodTerm const *term : terms) { + if (auto *sim = dynamic_cast(workspace.pdf(term->distribution.name))) { + sim->setAttribute(helperObjectTag); } - if (inDomains.count(variate) || arg->isConstant()) { - continue; + } + auto *result = static_cast(workspace.pdf(simPdfName)); + if (termData.empty()) { + return result; + } + + RooArgSet columnClones; // owns the clones; `vars` also lists the variables of the data sets + for (RooCategory const *column : columns) { + columnClones.addClone(*column); + } + RooArgSet vars{columnClones}; + bool weighted = false; + std::string weightName = RooFit::WeightVar().getString(0); + for (auto const *data : termData) { + vars.add(*(*data)->get(), /*silent=*/true); + weighted |= (*data)->isWeighted(); + if (auto *ds = dynamic_cast(data->get()); ds && ds->isWeighted()) { + weightName = ds->weightVar()->GetName(); + } + } + std::string const dataName = analysisName + "_data"; + std::unique_ptr merged; + if (weighted) { + RooRealVar weightVar{weightName.c_str(), weightName.c_str(), 1.0}; + vars.add(weightVar); + merged = std::make_unique(dataName.c_str(), dataName.c_str(), vars, RooFit::WeightVar(weightVar)); + } else { + merged = std::make_unique(dataName.c_str(), dataName.c_str(), vars); + } + // The entries of the merged array channels, combined into one per merged array. + std::map> mergedEntries; + RooArgSet row; + merged->get()->snapshot(row); + for (std::size_t i = 0; i < terms.size(); ++i) { + RooAbsData &data = **termData[i]; + for (int entry = 0; entry < data.numEntries(); ++entry) { + RooArgSet const &values = *data.get(entry); + const double weight = data.weight(); + std::size_t const c = channelOf(i, values); + if (auto found = mergedInto.find(c); found != mergedInto.end()) { + std::unique_ptr &combined = mergedEntries[found->second]; + if (!combined) { + combined = std::make_unique(); + values.snapshot(*combined); + continue; + } + // Identical factors have the same observed values in all data sets (HS3). + for (RooAbsArg const *value : values) { + auto const *real = dynamic_cast(value); + auto const *known = dynamic_cast(combined->find(value->GetName())); + if (real && known && !dynamic_cast(value) && + std::abs(real->getVal() - known->getVal()) > 1e-12 * std::max(1.0, std::abs(known->getVal()))) { + RooJSONFactoryWSTool::error("the observed value of '" + std::string(value->GetName()) + + "' differs between the data sets of the array distributions that " + + "share it in the likelihood of the analysis '" + analysisName + "'"); + } + if (!known) { + combined->addClone(*value); + } + } + continue; + } + row.assign(values); + for (RooCategory const *column : columns) { + auto &cat = static_cast(row[column->GetName()]); + if (column->GetName() == termColumn) { + cat.setLabel(terms[i]->distribution.name.c_str()); + } else if (!values.find(column->GetName())) { + cat.setLabel(""); + } + } + merged->add(row, weight); } - observables.add(*arg); - list += (list.empty() ? "'" : ", '") + variate + "'"; } - if (!list.empty()) { - oocoutI(nullptr, IO) << "RooFitHS3: no data set of the likelihood of the analysis '" << analysis.name - << "' gives its observables; they are the variates of the likelihood that are neither in a " - "domain of the analysis nor constant: " - << list << std::endl; + for (auto const &[c, values] : mergedEntries) { + row.assign(*values); + std::vector const key = keyOf(channels[c]); + for (std::size_t k = 0; k < columns.size(); ++k) { + static_cast(row[columns[k]->GetName()]).setLabel(key[k].c_str()); + } + merged->add(row, 1.0); + } + + // The observables refer to the variables of the data sets of the terms, which are replaced by the merged one. + RooArgSet mergedObservables; + for (RooAbsArg *obs : observables) { + RooAbsArg *inMerged = merged->get()->find(obs->GetName()); + mergedObservables.add(inMerged ? *inMerged : *obs, /*silent=*/true); + } + observables.removeAll(); + observables.add(mergedObservables); + + // The data sets of the terms are replaced, unless something else uses them. + std::size_t analysesOfLikelihood = 0; + for (HS3::Analysis const &other : doc.analyses) { + analysesOfLikelihood += other.likelihood.name == analysis.likelihood.name; + } + for (auto *data : termData) { + std::size_t uses = 0; + for (HS3::Likelihood const &likelihood : doc.likelihoods) { + for (HS3::LikelihoodTerm const &term : likelihood.terms) { + auto const *dataRef = std::get_if(&term.data); + uses += dataRef && dataRef->name == (*data)->GetName(); + } + } + if (uses == 1 && analysesOfLikelihood == 1) { + data->reset(); + } } + datasets.push_back(std::move(merged)); + return result; } +// Import an analysis (likelihood + domains) as one or more ModelConfig objects into the workspace. void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWorkspace &workspace, std::vector> &datasets, std::set const &bookkeepingColumns, - std::set const &variates) + std::set const &auxiliaryMeasurements) { - // if this is a toplevel pdf, also create a modelConfig for it std::string const &analysisName = analysis.name; auto foundInfo = doc.rooFit.modelConfigs.find(analysisName); HS3::ModelConfigInfo const *mcInfo = foundInfo != doc.rooFit.modelConfigs.end() ? &foundInfo->second : nullptr; @@ -1457,12 +1618,8 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo auto *mc = static_cast(workspace.obj(mcname)); mc->SetWS(workspace); - auto foundLikelihood = std::find_if(doc.likelihoods.begin(), doc.likelihoods.end(), - [&](HS3::Likelihood const &l) { return l.name == analysis.likelihood.name; }); - if (foundLikelihood == doc.likelihoods.end()) { - throw std::runtime_error("likelihood node not found!"); - } - HS3::Likelihood const &likelihood = *foundLikelihood; + // The validation has checked that the likelihood exists. + HS3::Likelihood const &likelihood = *HS3::findByName(doc.likelihoods, analysis.likelihood.name); auto checkPdf = [&](RooAbsArg const *arg, std::string const &name, std::string const &what) { requirePdf(arg, "the " + what + " '" + name + "' of analysis '" + analysisName + "'"); @@ -1515,16 +1672,18 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo [&](auto const &term) { return term.distribution.name == modelPdfName; })) { modelTree.reset(modelPdf->getComponents()); } + // So is a component of a combined distribution that is read as a constraint term (`auxiliaryMeasurements`, see + // readAuxiliaryChannelsAsConstraints()), whatever the document says about the ModelConfig. auto isConstraintTermOfModel = [&](RooAbsPdf const &listed, HS3::LikelihoodTerm const &term) { auto const *dataRef = std::get_if(&term.data); - if (!modelTree || !dataRef || modelTree->find(listed.GetName())) { + const bool auxiliary = auxiliaryMeasurements.count(listed.GetName()) > 0; + if (!dataRef || (!auxiliary && (!modelTree || modelTree->find(listed.GetName())))) { return false; } - return std::any_of(datasets.begin(), datasets.end(), - [&](auto const &d) { return d && d->GetName() == dataRef->name && d->numEntries() == 1; }); + auto const *data = findDataset(datasets, dataRef->name); + return data && (*data)->numEntries() == 1; }; - std::vector nllDistNames; std::vector nllTerms; RooArgSet constraintObservables; RooArgSet constraintsWithData; @@ -1534,38 +1693,33 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo if (listed && !isConstraintTermOfOthers(*listed, i) && isConstraintTermOfModel(*listed, term)) { constraintsWithData.add(*listed); } else if (!listed || !isConstraintTermOfOthers(*listed, i)) { - nllDistNames.push_back(term.distribution.name); nllTerms.push_back(&term); continue; } auto const *dataRef = std::get_if(&term.data); - if (!dataRef) { + auto *data = dataRef ? findDataset(datasets, dataRef->name) : nullptr; + if (!data) { continue; } - for (auto &d : datasets) { - if (!d || d->GetName() != dataRef->name) { + if ((*data)->numEntries() != 1) { + throw std::runtime_error("the data set '" + dataRef->name + "' paired with the constraint term '" + + term.distribution.name + "' must have exactly one entry"); + } + for (RooAbsArg const *obs : *(*data)->get(0)) { + auto *var = workspace.var(obs->GetName()); + auto const *value = dynamic_cast(obs); + if (!var || !value) { + RooJSONFactoryWSTool::warning("RooFitHS3: cannot set the observed value of '" + + std::string(obs->GetName()) + "' from the data set '" + dataRef->name + + "': it is not a variable of the workspace"); continue; } - if (d->numEntries() != 1) { - throw std::runtime_error("the data set '" + dataRef->name + "' paired with the constraint term '" + - term.distribution.name + "' must have exactly one entry"); - } - for (RooAbsArg const *obs : *d->get(0)) { - auto *var = workspace.var(obs->GetName()); - auto const *value = dynamic_cast(obs); - if (!var || !value) { - RooJSONFactoryWSTool::warning("RooFitHS3: cannot set the observed value of '" + - std::string(obs->GetName()) + "' from the data set '" + dataRef->name + - "': it is not a variable of the workspace"); - continue; - } - var->setVal(value->getVal()); - var->setConstant(true); - constraintObservables.add(*var); - } - // The values live in the variables now; the data set is not imported. - d.reset(); + var->setVal(value->getVal()); + var->setConstant(true); + constraintObservables.add(*var); } + // The values live in the variables now; the data set is not imported. + data->reset(); } // The constraint terms of HistFactory distributions are listed as auxiliary distributions, but they are part of the // distributions in RooFit. @@ -1584,8 +1738,8 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo checkPdf(extConstraint, aux.name, "auxiliary distribution"); // Only if the constraint really ended up in a distribution of the likelihood. if (channelConstraints.count(aux.name) && - std::any_of(nllDistNames.begin(), nllDistNames.end(), [&](std::string const &distName) { - RooAbsArg const *dist = workspace.arg(distName); + std::any_of(nllTerms.begin(), nllTerms.end(), [&](HS3::LikelihoodTerm const *term) { + RooAbsArg const *dist = workspace.arg(term->distribution.name); return dist && dist->dependsOn(*extConstraint); })) { continue; @@ -1596,22 +1750,14 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo extConstraints.add(*extConstraint); } RooArgSet observables; - for (HS3::LikelihoodTerm const *termPtr : nllTerms) { - HS3::LikelihoodTerm const &term = *termPtr; - auto const *dataRef = std::get_if(&term.data); - if (!dataRef) { - continue; - } - bool found = false; - for (const auto &d : datasets) { - // Components of combined data sets have been moved out of the list. - if (d && d->GetName() == dataRef->name) { - found = true; - observables.add(*d->get(), true); + for (HS3::LikelihoodTerm const *term : nllTerms) { + if (auto const *dataRef = std::get_if(&term->data)) { + auto const *data = findDataset(datasets, dataRef->name); + if (!data) { + throw std::runtime_error("dataset '" + dataRef->name + "' cannot be found!"); } + observables.add(*(*data)->get(), true); } - if (!found) - throw std::runtime_error("dataset '" + dataRef->name + "' cannot be found!"); } // Bookkeeping columns of the data sets are no observables of the model. for (std::string const &column : bookkeepingColumns) { @@ -1620,7 +1766,7 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo } } if (observables.empty()) { - observablesWithoutData(doc, analysis, mcInfo, variates, workspace, observables); + observablesWithoutData(doc, analysis, likelihood, mcInfo, workspace, observables); } // A combined distribution paired with a combined data set must have a sub-data set for every component. @@ -1630,42 +1776,32 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo if (!dataRef || !simPdf) { continue; } - for (HS3::Data const &data : doc.data) { - auto const *combined = std::get_if(&data.body); - if (combined && data.name == dataRef->name) { - checkCombinedPairing(*simPdf, data.name, *combined); - } + if (auto const *combined = HS3::find(doc.data, dataRef->name)) { + checkCombinedPairing(*simPdf, dataRef->name, *combined); } } - std::string const pdfName = mcInfo && mcInfo->pdfName ? *mcInfo->pdfName : "simPdf"; - - RooAbsPdf *pdf = workspace.pdf(pdfName); - - if (!pdf) { - // if there is no simultaneous pdf, we can check whether there is only one pdf in the list - if (nllDistNames.size() == 1) { - // if so, we can use that one to populate the ModelConfig - pdf = workspace.pdf(nllDistNames[0]); - } else { - // otherwise, we have no choice but to build a simPdf by hand - std::string simPdfName = analysisName + "_simPdf"; - std::string indexCatName = analysisName + "_categoryIndex"; - RooCategory indexCat{indexCatName.c_str(), indexCatName.c_str()}; - std::map pdfMap; - for (std::size_t i = 0; i < nllDistNames.size(); ++i) { - indexCat.defineType(nllDistNames[i], i); - pdfMap[nllDistNames[i]] = workspace.pdf(nllDistNames[i]); - } - RooSimultaneous simPdf{simPdfName.c_str(), simPdfName.c_str(), pdfMap, indexCat}; - workspace.import(simPdf, RooFit::RecycleConflictNodes(true), RooFit::Silence(true)); - pdf = workspace.pdf(simPdfName); + // The pdf of the ModelConfig is the one named in the RooFit metadata, e.g. in files of ROOT before HS3 0.3 the + // RooSimultaneous of the channels that the likelihood lists. Otherwise, it is the only term of the likelihood, or + // the union of its terms. Without terms, it is "simPdf". + RooAbsPdf *pdf = mcInfo && mcInfo->pdfName ? workspace.pdf(*mcInfo->pdfName) : nullptr; + if (!pdf && !nllTerms.empty()) { + pdf = flatModelOfLikelihood(doc, analysis, workspace, nllTerms, datasets, observables); + if (!pdf) { + pdf = workspace.pdf(nllTerms.front()->distribution.name); } + } else if (!pdf) { + pdf = workspace.pdf("simPdf"); + } + if (!pdf) { + RooJSONFactoryWSTool::error("the likelihood '" + likelihood.name + "' of analysis '" + analysisName + + "' has no distribution"); } mc->SetPdf(*pdf); extConstraints.add(constraintsWithData, /*silent=*/true); + attachConstraintsToChannels(*pdf, extConstraints); if (!extConstraints.empty()) mc->SetExternalConstraints(extConstraints); @@ -1713,14 +1849,12 @@ void importAnalysis(HS3Document const &doc, HS3::Analysis const &analysis, RooWo RooArgSet domainPars; for (auto const &domainRef : analysis.domains) { - auto thisDomain = std::find_if(doc.domains.begin(), doc.domains.end(), - [&](HS3::Domain const &d) { return d.name == domainRef.name; }); - if (thisDomain == doc.domains.end()) - continue; - for (auto const &axis : thisDomain->axes) { - auto *wsvar = workspace.var(std::visit([](auto const &a) -> std::string const & { return a.name; }, axis)); - if (wsvar) - domainPars.add(*wsvar); + if (HS3::Domain const *domain = HS3::findByName(doc.domains, domainRef.name)) { + for (auto const &axis : domain->axes) { + if (RooRealVar *wsvar = workspace.var(HS3::nameOf(axis))) { + domainPars.add(*wsvar); + } + } } } @@ -1834,9 +1968,8 @@ void combineDatasets(HS3Document const &doc, std::vectorGetName() == componentName; }); - if (found == datasets.end()) { + auto *found = findDataset(datasets, componentName); + if (!found) { RooJSONFactoryWSTool::error("unable to obtain component matching component name '" + componentName + "'"); } allVars.add(*(*found)->get(), true); @@ -1850,49 +1983,88 @@ void combineDatasets(HS3Document const &doc, std::vector +std::vector +columnCategories(RooWorkspace const &workspace, HS3::Combined const &combined, std::string const &context) +{ + std::vector columns; + for (std::string const &column : combined.columns) { + RooCategory *cat = workspace.cat(column); + if (!cat) { + RooJSONFactoryWSTool::error("the column '" + column + "' of " + context + + " is not declared by a categorical axis of a domain"); + } + columns.push_back(cat); + } + for (auto const &component : combined.components) { + for (std::size_t i = 0; i < columns.size(); ++i) { + if (!columns[i]->hasLabel(component.key[i])) { + RooJSONFactoryWSTool::error("the label '" + component.key[i] + "' in position " + std::to_string(i) + + " of a key of " + context + " is not one of the categories of '" + + columns[i]->GetName() + "'"); + } + } + } + return columns; +} + // Build the combined data sets of the document: a RooDataSet with one category column per entry of "columns" and the // variables of all components, filled row by row from the components, which are then no longer imported on their own. -// The columns are categories declared by categorical axes of the domains. A key without a sub-data set is a state -// without entries: RooFit has no notion of an absent component. The validation has checked the columns, the keys and -// that the components are data sets of the document. +// A key without a sub-data set is a state without entries: RooFit has no notion of an absent component. The validation +// has checked that the components are data sets of the document. +// +// A component may itself be a combined data set (HS3 #161), which is then built first. Its columns become columns of +// the outer data set, like the RooSimultaneous of a combined distribution with a combined component is flattened: the +// rows of the other components have the label "" in them. void assembleCombinedData(HS3Document const &doc, RooWorkspace const &workspace, std::vector> &datasets) { + // The combined data sets in an order in which the components come first. + std::vector pending; for (HS3::Data const &input : doc.data) { - auto const *info = std::get_if(&input.body); - if (!info) { - continue; + if (std::holds_alternative(input.body)) { + pending.push_back(&input); } + } + std::vector ordered; + std::set done; + while (!pending.empty()) { + auto ready = std::find_if(pending.begin(), pending.end(), [&](HS3::Data const *data) { + for (auto const &c : std::get(data->body).components) { + if (HS3::find(doc.data, c.ref.name) && !done.count(c.ref.name)) { + return false; + } + } + return true; + }); + if (ready == pending.end()) { + RooJSONFactoryWSTool::error("the combined data set '" + pending.front()->name + + "' is a component of itself, directly or through other combined data sets"); + } + ordered.push_back(*ready); + done.insert((*ready)->name); + pending.erase(ready); + } + for (HS3::Data const *inputPtr : ordered) { + HS3::Data const &input = *inputPtr; + auto const *info = std::get_if(&input.body); std::string const &name = input.name; RooJSONFactoryWSTool::testValidName(name, true); std::string const context = "the combined data set '" + name + "'"; - std::vector columns; - for (std::string const &column : info->columns) { - RooCategory *cat = workspace.cat(column); - if (!cat) { - RooJSONFactoryWSTool::error("the column '" + column + "' of " + context + - " is not declared by a categorical axis of a domain"); - } - columns.push_back(cat); - } + std::vector columns = columnCategories(workspace, *info, context); struct Component { std::vector const *key; std::unique_ptr data; }; std::vector components; - for (HS3::CombinedDataComponent const &c : info->components) { - for (std::size_t i = 0; i < columns.size(); ++i) { - if (!columns[i]->hasLabel(c.key[i])) { - RooJSONFactoryWSTool::error("the label '" + c.key[i] + "' in position " + std::to_string(i) + - " of a key of " + context + " is not one of the categories of '" + - columns[i]->GetName() + "'"); - } - } - auto found = - std::find_if(datasets.begin(), datasets.end(), [&](auto &d) { return d && d->GetName() == c.data.name; }); - if (found == datasets.end()) { - RooJSONFactoryWSTool::error("the component '" + c.data.name + "' of " + context + + for (auto const &c : info->components) { + auto *found = findDataset(datasets, c.ref.name); + if (!found) { + RooJSONFactoryWSTool::error("the component '" + c.ref.name + "' of " + context + " is not a data set of the document, or it is already a component of " + "another combined data set"); } @@ -1905,6 +2077,31 @@ void assembleCombinedData(HS3Document const &doc, RooWorkspace const &workspace, for (RooCategory const *column : columns) { columnClones.addClone(*column); } + // The columns of components that are combined data sets themselves, with the label "" for the other rows. + for (auto const &c : components) { + for (RooAbsArg const *arg : *c.data->get()) { + if (dynamic_cast(arg) && !columnClones.find(*arg)) { + // The category of the domain has the labels of all components. + RooCategory const *declared = workspace.cat(arg->GetName()); + columnClones.addClone(declared ? static_cast(*declared) : *arg); + } + } + } + std::vector innerColumns; + for (RooAbsArg *arg : columnClones) { + auto *cat = static_cast(arg); + if (std::find_if(columns.begin(), columns.end(), [&](RooCategory const *c) { + return std::strcmp(c->GetName(), cat->GetName()) == 0; + }) != columns.end()) { + continue; + } + innerColumns.push_back(cat); + bool everywhere = std::all_of(components.begin(), components.end(), + [&](auto const &c) { return c.data->get()->find(*cat) != nullptr; }); + if (!everywhere && !cat->hasLabel("")) { + cat->defineType(""); + } + } RooArgSet vars{columnClones}; bool weighted = false; bool storeErrors = false; @@ -1933,6 +2130,11 @@ void assembleCombinedData(HS3Document const &doc, RooWorkspace const &workspace, } for (int i = 0; i < c.data->numEntries(); ++i) { vars.assign(*c.data->get(i)); + for (RooCategory *inner : innerColumns) { + if (!c.data->get()->find(*inner)) { + static_cast(vars[inner->GetName()]).setLabel(""); + } + } if (weighted) { combined->add(vars, c.data->weight(), storeErrors ? c.data->weightError(RooAbsData::SumW2) : 0.0); } else { @@ -1950,23 +2152,36 @@ void sortByName(T &coll) std::sort(coll.begin(), coll.end(), [](auto &l, auto &r) { return strcmp(l->GetName(), r->GetName()) < 0; }); } -/// Check whether an object is a top-level object of the workspace, i.e. not a -/// component of some other workspace object. Only clients that are part of the -/// workspace count: RooFit also registers evaluation artifacts as clients, -/// like the normalization integral that RooAbsPdf caches after -/// getVal(normSet), or the integral returned by createIntegral(). These live -/// outside the workspace and are not evidence that the object is a sub-node of -/// a bigger model (see https://github.com/root-project/root/issues/23221). +// Whether no object of the workspace uses `arg`. Only clients that are part of the workspace count: RooFit also +// registers evaluation artifacts as clients, like the normalization integral that RooAbsPdf caches after +// getVal(normSet), or the integral returned by createIntegral(). These live outside the workspace and are not evidence +// that the object is a sub-node of a bigger model (see https://github.com/root-project/root/issues/23221). Helpers that +// are not written don't count as users either. bool isTopLevel(RooAbsArg const &arg, RooWorkspace const &ws) { for (RooAbsArg const *client : arg.clients()) { - if (ws.components().containsInstance(*client)) { + if (ws.components().containsInstance(*client) && !isSkippedHelper(*client, ws)) { return false; } } return true; } +// The top-level objects among `args` that are written, sorted by name. +template +std::vector topLevelObjects(RooArgSet const &args, RooWorkspace const &ws) +{ + std::vector out; + for (RooAbsArg *arg : args) { + auto *object = dynamic_cast(arg); + if (object && isTopLevel(*arg, ws) && !isSkippedHelper(*arg, ws)) { + out.push_back(object); + } + } + sortByName(out); + return out; +} + } // namespace RooArgSet RooFit::HS3::createObservables(std::vector const &axes) @@ -1986,10 +2201,14 @@ std::unique_ptr RooFit::HS3::createDataHist(std::string const &name RooJSONFactoryWSTool::error("inconsistent number of errors and contents in '" + name + "': errors=" + std::to_string(errors->size()) + ", contents=" + std::to_string(contents.size())); } - auto bins = generateBinIndices(vars); - if (contents.size() != bins.size()) { + // A continuous variable has the bins of its binning, and a category one bin per state. + std::size_t nBins = 1; + for (RooAbsArg const *var : vars) { + nBins *= std::max(0, dynamic_cast(*var).numBins()); + } + if (contents.size() != nBins) { RooJSONFactoryWSTool::error("inconsistent bin numbers: contents=" + std::to_string(contents.size()) + - ", bins=" + std::to_string(bins.size())); + ", bins=" + std::to_string(nBins)); } auto dh = std::make_unique(name, name, vars); // RooDataHist only starts to store squared weights when it sees a positive error, and then initializes them from @@ -1997,16 +2216,137 @@ std::unique_ptr RooFit::HS3::createDataHist(std::string const &name if (errors && !errors->empty()) { dh->set(0, 0., 1.); } - for (std::size_t ibin = 0; ibin < bins.size(); ++ibin) { + for (std::size_t ibin = 0; ibin < nBins; ++ibin) { dh->set(ibin, contents[ibin], errors ? (*errors)[ibin] : -1); } return dh; } +// A combined distribution is a RooSimultaneous. Its index category is the single column, or a RooSuperCategory over the +// columns in the order of the document. +void RooFit::HS3::buildCombinedDist(RooJSONFactoryWSTool &tool, std::string const &name, CombinedDist const &dist, + std::vector const &otherLikelihoodTerms) +{ + std::vector columns = columnCategories(*tool.workspace(), dist, "the combined_dist '" + name + "'"); + std::map pdfMap; + for (auto const &component : dist.components) { + pdfMap[selectionLabel(component.key)] = tool.request(component.ref.name, name); + } + + // In RooFit, the constraint terms of a channel are factors of it, which RooFit takes out of the product when it + // builds a likelihood. A channel that is a product gets them when the likelihood is read; the other channels are + // wrapped in a product "_with_constraints" here, since the components of a RooSimultaneous can't be + // replaced later. The candidates are the other terms of the likelihoods that have this distribution. + std::vector terms; + for (std::string const &term : otherLikelihoodTerms) { + if (auto *pdf = tool.tryRequest(term)) { + terms.push_back(pdf); + } + } + RooArgSet channels; + for (auto const &item : pdfMap) { + channels.add(*item.second, /*silent=*/true); + } + for (auto &item : pdfMap) { + RooAbsPdf &channel = *item.second; + RooArgList factors; + for (RooAbsPdf *term : terms) { + // A combined distribution as a component is flattened by the import of the likelihood (HS3 #161); its + // channels get their constraints there. + if (!dynamic_cast(&channel) && !dynamic_cast(&channel) && + isConstraintOfChannel(*term, channel, channels)) { + factors.add(*term); + } + } + if (!factors.empty()) { + factors.add(channel); + item.second = &tool.wsEmplace(std::string(channel.GetName()) + "_with_constraints", factors); + } + } + + if (columns.size() == 1) { + tool.wsImport(RooSimultaneous{name.c_str(), name.c_str(), pdfMap, *columns.front()}); + return; + } + RooArgSet inputs; + for (RooCategory *column : columns) { + inputs.add(*column); + } + std::string const indexName = name + "_index"; + RooSuperCategory indexCat{indexName.c_str(), indexName.c_str(), inputs}; + tool.wsImport(RooSimultaneous{name.c_str(), name.c_str(), pdfMap, indexCat}); +} + +bool isSkippedHelper(RooAbsArg const &arg, RooWorkspace const &ws) +{ + const bool helper = arg.getAttribute(helperObjectTag); + // An object that nothing in the workspace uses: written, unless it is a helper. A helper that stands for an HS3 + // object of its own, an array distribution, is written as that object. + bool hasClients = false; + for (RooAbsArg const *client : arg.clients()) { + if (!ws.components().containsInstance(*client)) { + continue; + } + const bool skipped = isSkippedHelper(*client, ws); + if (!skipped) { + return false; + } + // An array distribution that is not written dissolves into its components, which are then written on their + // own: it is no user of them. + if (!(client->getAttribute(helperObjectTag) && client->getAttribute(arrayDistTag))) { + hasClients = true; + } + } + if (!hasClients && (!helper || (arg.getAttribute(arrayDistTag) && RooJSONFactoryWSTool::config().exportArrayDist))) { + return false; + } + for (TObject const *obj : ws.allGenericObjects()) { + auto const *mc = dynamic_cast(obj); + if (!mc) { + continue; + } + if (mc->GetPdf() == &arg || mc->GetPriorPdf() == &arg || + (mc->GetExternalConstraints() && mc->GetExternalConstraints()->containsInstance(arg))) { + return false; + } + } + // A helper that only skipped helpers use, or nothing; and, weakly transitive, any other object that only skipped + // helpers use. + return true; +} + RooJSONFactoryWSTool::RooJSONFactoryWSTool(RooWorkspace &ws) : _workspace{ws} {} +bool RooJSONFactoryWSTool::isConstraintReadAsChannel(RooAbsPdf const &pdf) const +{ + if (!_import) { + return false; + } + auto const &componentsWithData = _import->componentsWithData; + // Also as a component of an array distribution, which is the product of its components in RooFit. + return componentsWithData.count(pdf.GetName()) || + std::any_of(pdf.clients().begin(), pdf.clients().end(), [&](RooAbsArg const *client) { + return client->getAttribute(arrayDistTag) && componentsWithData.count(client->GetName()); + }); +} + +std::vector const *RooJSONFactoryWSTool::channelConstraints(std::string const &dist) const +{ + if (!_import) { + return nullptr; + } + auto const &all = _import->document.rooFit.channelConstraints; + auto found = all.find(dist); + return found != all.end() ? &found->second : nullptr; +} + RooJSONFactoryWSTool::~RooJSONFactoryWSTool() = default; +HS3::Version RooJSONFactoryWSTool::hs3Version() const +{ + return _import ? _import->document.metadata.version : HS3::Version{}; +} + void RooJSONFactoryWSTool::fillSeq(JSONNode &node, RooAbsCollection const &coll, size_t nMax) { const size_t old_children = node.num_children(); @@ -2042,32 +2382,12 @@ JSONNode const *RooJSONFactoryWSTool::findNamedChild(JSONNode const &node, std:: return nullptr; } -/** - * @brief Check if a string is a valid name. - * - * A valid name should start with a letter or an underscore, followed by letters, digits, or underscores. - * Only characters from the ASCII character set are allowed. - * - * @param str The string to be checked. - * @return bool Returns true if the string is a valid name; otherwise, returns false. - */ +/// Whether a string is a valid name: a letter or an underscore, followed by letters, digits or underscores. Only +/// characters from the ASCII character set are allowed. bool RooJSONFactoryWSTool::isValidName(const std::string &str) { - // Check if the string is empty or starts with a non-letter/non-underscore character - if (str.empty() || !(std::isalpha(str[0]) || str[0] == '_')) { - return false; - } - - // Check the remaining characters in the string - for (char c : str) { - // Allow letters, digits, and underscore - if (!(std::isalnum(c) || c == '_')) { - return false; - } - } - - // If all characters are valid, the string is a valid name - return true; + return !str.empty() && (std::isalpha(str[0]) || str[0] == '_') && + std::all_of(str.begin(), str.end(), [](char c) { return std::isalnum(c) || c == '_'; }); } bool RooJSONFactoryWSTool::testValidName(const std::string &name, bool forceError) @@ -2091,48 +2411,37 @@ std::string RooJSONFactoryWSTool::name(const JSONNode &n) return n["name"].val(); } -JSONNode &RooJSONFactoryWSTool::makeVariablesNode(JSONNode &rootNode) -{ - return appendNamedChild(rootNode["parameter_points"], "default_values")["parameters"]; -} - +// The specializations of requestImpl() look in the workspace first. If the object is not there, they import what the +// document has under that name, and look again. template <> RooRealVar *RooJSONFactoryWSTool::requestImpl(const std::string &objname) { - if (RooRealVar *retval = _workspace.var(objname)) - return retval; - if (_import) { - auto it = _import->defaultValuesByName.find(objname); - if (it != _import->defaultValuesByName.end()) { - this->importVariable(*it->second); - if (RooRealVar *retval = _workspace.var(objname)) - return retval; + if (_import && !_workspace.var(objname)) { + auto found = _import->defaultValuesByName.find(objname); + if (found != _import->defaultValuesByName.end()) { + importVariable(*found->second); } } - return nullptr; + return _workspace.var(objname); } template <> RooAbsPdf *RooJSONFactoryWSTool::requestImpl(const std::string &objname) { - if (RooAbsPdf *retval = _workspace.pdf(objname)) - return retval; - if (_import) { - auto it = _import->distributions.find(objname); - if (it != _import->distributions.end()) { - this->importDistribution(*it->second); - if (RooAbsPdf *retval = _workspace.pdf(objname)) - return retval; + if (_import && !_workspace.pdf(objname)) { + auto found = _import->distributions.find(objname); + if (found != _import->distributions.end()) { + importDistribution(*found->second); } - // HS3 lists some resolution models, which are pdfs in RooFit, as functions. - auto function = _import->functions.find(objname); - if (function != _import->functions.end()) { - this->importFunction(*function->second->body.node, true); - if (RooAbsPdf *retval = _workspace.pdf(objname)) - return retval; + } + // HS3 lists some resolution models, which are pdfs in RooFit, as functions. + if (_import && !_workspace.pdf(objname)) { + auto found = _import->functions.find(objname); + if (found != _import->functions.end()) { + importFunction(*found->second->body.node, true); } } - return nullptr; + return _workspace.pdf(objname); } template <> @@ -2140,28 +2449,23 @@ RooAbsReal *RooJSONFactoryWSTool::requestImpl(const std::string &obj { if (RooAbsReal *retval = _workspace.function(objname)) return retval; - auto toNumOutput = toNumber(objname); - if (toNumOutput.valid) - return &RooFit::RooConst(toNumOutput.value); + if (std::optional number = toNumber(objname)) + return &RooFit::RooConst(*number); if (RooAbsPdf *pdf = requestImpl(objname)) return pdf; if (RooRealVar *var = requestImpl(objname)) return var; - if (RooAbsReal *retval = _workspace.function(objname)) - return retval; - if (_import) { - auto it = _import->functions.find(objname); - if (it != _import->functions.end()) { - this->importFunction(*it->second->body.node, true); - if (RooAbsReal *retval = _workspace.function(objname)) - return retval; + if (_import && !_workspace.function(objname)) { + auto found = _import->functions.find(objname); + if (found != _import->functions.end()) { + importFunction(*found->second->body.node, true); } } - return nullptr; + return _workspace.function(objname); } // Export a single variable (RooRealVar or RooConstVar) `v` to the parameter point. -void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, HS3::ParameterPoint &point, bool storeConstant) +void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, HS3::ParameterPoint &point) { auto *cv = dynamic_cast(v); auto *rrv = dynamic_cast(v); @@ -2186,32 +2490,33 @@ void RooJSONFactoryWSTool::exportVariable(const RooAbsArg *v, HS3::ParameterPoin HS3::ParameterValue var; var.name = v->GetName(); var.value = rrv->getVal(); - var.isConst = storeConstant && (rrv->isConstant() || rrv->getMin() >= rrv->getMax()); + var.isConst = rrv->isConstant() || rrv->getMin() >= rrv->getMax(); recordDomains(_export->domains, *rrv); point.parameters.push_back(std::move(var)); } -// Export all variables in `allElems` to the parameter point. -void RooJSONFactoryWSTool::exportVariables(const RooArgSet &allElems, HS3::ParameterPoint &point, bool storeConstant) -{ - for (RooAbsArg *arg : allElems) { - exportVariable(arg, point, storeConstant); - } -} - std::string RooJSONFactoryWSTool::exportTransformed(const RooAbsReal *original, const std::string &suffix, const std::string &formula) { - std::string newname = std::string(original->GetName()) + suffix; + // A literal constant is spelled as an identifier, which a parameter must not be mistaken for when reading. + if (!isLiteralConstVar(*original) && isLiteralIdentifier(original->GetName())) { + error("cannot write a function of '" + std::string(original->GetName()) + + "': names of the form 'const_' are reserved for functions of literal constants"); + } + std::string newname = transformedBaseName(*original) + suffix; + // HS3 0.3 (HS3 PR #154) only allows expressions to refer to parameters, variables and functions. + if (dynamic_cast(original)) { + error("the parameter '" + std::string(original->GetName()) + + "' would have to be written as an expression that refers to a pdf, which HS3 doesn't allow"); + } // Another object that shares the parameter might have exported the helper already. auto &functions = _export->doc.functions; - if (std::find_if(functions.begin(), functions.end(), [&](auto const &f) { return f.name == newname; }) != - functions.end()) { + if (HS3::findByName(functions, newname)) { return newname; } RooFit::Detail::JSONNode &trafo_node = appendNamedChild((*_export->root)["functions"], newname); trafo_node["type"] << "generic"; - trafo_node["expression"] << TString::Format(formula.c_str(), original->GetName()).Data(); + trafo_node["expression"] << TString::Format(formula.c_str(), transformedArgument(*original).c_str()).Data(); functions.push_back({newname, {"generic", nullptr}}); this->setAttribute(newname, "roofit_skip"); // this function should not be imported back in return newname; @@ -2235,26 +2540,17 @@ std::string makeValidNameOrError(std::string const &in) } // The labels of a selection key, made valid names and joined by "_". Used in the names of sub-data sets and of -// renamed observables. +// renamed observables. An empty label, for a column that does not apply to a component, stays empty, so that the +// position of the other labels still tells the keys apart. std::string selectionKeyName(std::vector const &key) { std::string out; - for (std::string const &label : key) { - out += (out.empty() ? "" : "_") + makeValidNameOrError(label); + for (std::size_t i = 0; i < key.size(); ++i) { + out += (i == 0 ? "" : "_") + (key[i].empty() ? std::string{} : makeValidNameOrError(key[i])); } return out; } -// The names of the category columns of `vars`. -std::vector selectionColumnNames(RooAbsCollection const &vars) -{ - std::vector names; - for (RooCategory const *column : selectionColumns(vars)) { - names.push_back(column->GetName()); - } - return names; -} - // The RooSimultaneous of the workspace (the first by name) whose columns are `columns`, or nullptr. RooSimultaneous const *findMatchingSimultaneous(RooWorkspace const &workspace, std::vector const &columns) { @@ -2266,11 +2562,7 @@ RooSimultaneous const *findMatchingSimultaneous(RooWorkspace const &workspace, s } sortByName(candidates); for (RooSimultaneous const *candidate : candidates) { - std::vector candidateColumns; - for (RooCategory const *column : selectionColumns(candidate->indexCat())) { - candidateColumns.push_back(column->GetName()); - } - if (candidateColumns == columns) { + if (namesOf(selectionColumns(candidate->indexCat())) == columns) { return candidate; } } @@ -2281,7 +2573,8 @@ RooSimultaneous const *findMatchingSimultaneous(RooWorkspace const &workspace, s namespace { -// Record an object that was written by an exporter in the document. +// Record an object that was written by an exporter in the document. Its node moves when the array that it is in +// grows, so the distributions only get their nodes at the end of the export. void recordGenericObject(HS3Document &doc, bool isPdf, std::string const &name, JSONNode const &elem) { std::string type = elem.has_child("type") ? elem["type"].val() : ""; @@ -2301,7 +2594,6 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func) std::string name = func.GetName(); std::set &exportedObjectNames = _export->exportedObjectNames; - // if this element was already exported, skip if (exportedObjectNames.find(name) != exportedObjectNames.end()) { // A component of an array distribution is not written under its own name, so an object that refers to it by // that name would have a dangling reference. @@ -2334,7 +2626,7 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func) // categorical axes of the domain that its "columns" name. return; } else if (dynamic_cast(&func) || dynamic_cast(&func)) { - exportVariable(&func, _export->doc.parameterPoints.front(), true); + exportVariable(&func, _export->doc.parameterPoints.front()); return; } else if (config().exportArrayDist && func.getAttribute(arrayDistTag)) { // An array distribution (HS3 `array_dist`): its components are registered here, and the array is written at @@ -2346,10 +2638,21 @@ void RooJSONFactoryWSTool::exportObject(RooAbsArg const &func) } for (RooAbsPdf *pdf : static_range_cast(prod->pdfList())) { const std::string writtenIn = exportAsArrayComponent(*pdf, name); - if (writtenIn != name) { - error("cannot write '" + name + "' as an array distribution: its component '" + pdf->GetName() + - "' is already written as '" + writtenIn + "'"); + if (writtenIn == name) { + continue; } + // A component of another array distribution: HS3 composes array distributions by union, so a component + // written in several of them is one factor. It is written in this one as well. + if (writtenIn != pdf->GetName() && _export->arrayComponents.count(writtenIn)) { + auto &components = _export->arrayComponents[name]; + if (components.empty()) { + _export->exportedObjectNames.insert(name); + } + components.push_back(pdf); + continue; + } + error("cannot write '" + name + "' as an array distribution: its component '" + pdf->GetName() + + "' is already written as '" + writtenIn + "'"); } return; } @@ -2457,7 +2760,7 @@ void RooJSONFactoryWSTool::exportArrayDists() // Exporting the servers may add to the registry, so the components are copied. const std::vector components = next->second; JSONNode &elem = appendNamedChild((*_export->root)["distributions"], arrayName); - writeArrayDist(elem, arrayName, components, _export->constraintObservables); + writeArrayDist(*this, elem, arrayName, components); recordGenericObject(_export->doc, true, arrayName, elem); for (RooAbsPdf const *pdf : components) { exportObjects(pdf->servers()); @@ -2483,20 +2786,24 @@ void RooJSONFactoryWSTool::recordConstraintTerms(std::string const &pdfName, std { auto &out = _export->constraintTermsOfPdf[pdfName]; for (auto const &term : terms) { - if (std::find(out.begin(), out.end(), term) == out.end()) { - out.push_back(term); - } + addUnique(out, term); } } -// Write the data set "_data" with the observed values of the observables of the constraint term `term` (an +// Write the data set `dataName` with the observed values of the observables of the constraint term `term` (an // array distribution written in this export, or a single constraint pdf) and return true, or return false if the // observables of the term are not all global observables of the model. -bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgSet const *globalObservables) +bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgSet const *globalObservables, + std::string const &dataName) { std::vector components; if (auto found = _export->arrayComponents.find(term); found != _export->arrayComponents.end()) { components = found->second; + } else if (auto const *product = dynamic_cast(_workspace.pdf(term))) { + // A product of constraint terms: its factors. + for (RooAbsArg const *factor : product->pdfList()) { + components.push_back(static_cast(factor)); + } } else if (RooAbsPdf *pdf = _workspace.pdf(term)) { components.push_back(pdf); } @@ -2511,7 +2818,6 @@ bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgS if (observables.empty()) { return false; } - const std::string dataName = term + "_data"; if (!_export->writtenConstraintData.insert(dataName).second) { return true; } @@ -2525,22 +2831,23 @@ bool RooJSONFactoryWSTool::exportConstraintData(std::string const &term, RooArgS if (!var) { error("the observable '" + name + "' of the constraint term '" + term + "' is not in the workspace"); } + // A range without a bound leaves the bound out, which means that the axis is unbounded there. HS3::ContinuousAxis axis; axis.name = name; - // A range without a bound gets the largest representable value, so that other engines see a finite axis. - axis.min = -std::numeric_limits::max(); - axis.max = std::numeric_limits::max(); if (auto const *rrv = dynamic_cast(var)) { - if (std::isfinite(rrv->getMin())) { - axis.min = rrv->getMin(); - } - if (std::isfinite(rrv->getMax())) { - axis.max = rrv->getMax(); - } + axis = toContinuousAxis(*rrv); } data.axes.emplace_back(std::move(axis)); data.entries.push_back(var->getVal()); } + // The observed value of a single observable is point data. + if (data.axes.size() == 1) { + HS3::PointData point; + point.value = data.entries.front(); + point.axes.push_back(std::get(data.axes.front())); + _export->doc.data.push_back({dataName, std::move(point)}); + return true; + } _export->doc.data.push_back({dataName, std::move(data)}); return true; } @@ -2558,14 +2865,13 @@ void RooJSONFactoryWSTool::exportSimultaneous(RooSimultaneous const &simPdf) RooAbsCategory const &indexCat = simPdf.indexCat(); std::vector columns = selectionColumns(indexCat); - JSONNode &elem = appendNamedChild((*_export->root)["distributions"], simPdf.GetName()); - elem["type"] << "combined_dist"; - auto &columnsNode = elem["columns"].set_seq(); + // The distribution takes its place among the distributions here, and is written at the end of the export. + appendNamedChild((*_export->root)["distributions"], simPdf.GetName()); + HS3::CombinedDist combined; for (RooCategory const *column : columns) { - columnsNode.append_child() << column->GetName(); + combined.columns.push_back(column->GetName()); recordCategory(*column); } - auto &components = elem["components"].set_seq(); // Components with renamed observables replace the original ones (Config::renameCombinedObservables). auto renamed = _renamedComponents.find(simPdf.GetName()); std::set replaced; @@ -2587,11 +2893,9 @@ void RooJSONFactoryWSTool::exportSimultaneous(RooSimultaneous const &simPdf) replacements.push_back(pdf); } } - JSONNode &component = components.append_child().set_map(); - component["key"].fill_seq(key); - component["dist"] << pdf->GetName(); + combined.components.push_back({key, {pdf->GetName()}}); } - recordGenericObject(_export->doc, true, simPdf.GetName(), elem); + _export->doc.distributions.push_back({simPdf.GetName(), std::move(combined)}); std::vector servers; for (RooAbsArg const *server : simPdf.servers()) { @@ -2608,11 +2912,7 @@ void RooJSONFactoryWSTool::exportSimultaneous(RooSimultaneous const &simPdf) // document is read back. void RooJSONFactoryWSTool::recordCategory(RooAbsCategory const &cat) { - HS3::CategoricalAxis axis; - axis.name = cat.GetName(); - for (unsigned int i = 0; i < cat.size(); ++i) { - axis.categories.push_back(cat.getOrdinal(i).first); - } + HS3::CategoricalAxis axis = toCategoricalAxis(cat); auto found = _export->categories.find(axis.name); if (found != _export->categories.end() && found->second.categories != axis.categories) { RooJSONFactoryWSTool::error("the category '" + axis.name + @@ -2621,16 +2921,9 @@ void RooJSONFactoryWSTool::recordCategory(RooAbsCategory const &cat) _export->categories[axis.name] = std::move(axis); } -/** - * @brief Import a function from the JSONNode into the workspace. - * - * This function imports a function from the given JSONNode into the workspace. - * The function's information is read from the JSONNode and added to the workspace. - * - * @param p The JSONNode representing the function to be imported. - * @param importAllDependants A boolean flag indicating whether to import all dependants (servers) of the function. - * @return void - */ +/// Import the function or distribution `p` into the workspace, with the importers that are registered for its type. +/// If `importAllDependants` is false, `p` is a fragment of a document, whose own "functions" and "distributions" are +/// imported first. void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDependants) { // To report the components that no importer understands. @@ -2644,15 +2937,12 @@ void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDepen auto const &importers = RooFit::JSONIO::importers(); - // some preparations: what type of function are we dealing with here? RooJSONFactoryWSTool::testValidName(name, true); - // if the RooAbsArg already exists, we don't need to do anything if (_workspace.arg(name)) { return; } ImportGuard importGuard{_importStack, name}; - // if the key we found is not a map, it's an error if (!p.is_map()) { RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() function node " + name + " is not a map!"); } @@ -2670,20 +2960,11 @@ void RooJSONFactoryWSTool::importFunction(const JSONNode &p, bool importAllDepen this->importDependants(p); } - // check for specific implementations auto it = importers.find(functype); bool ok = false; if (it != importers.end()) { for (auto &imp : it->second) { - // The built-in importer doesn't see the constraint terms that the document adds to the channel. - if (_import && HS3::isBuiltinHistFactoryImporter(*imp)) { - auto const &channelConstraints = _import->document.rooFit.channelConstraints; - auto found = channelConstraints.find(name); - HS3::importHistFactoryNode(*this, p, found != channelConstraints.end() ? &found->second : nullptr); - ok = true; - } else { - ok = imp->importArg(this, p); - } + ok = imp->importArg(this, p); if (ok) break; } @@ -2719,105 +3000,31 @@ void RooJSONFactoryWSTool::importDistribution(HS3::Distribution const &distribut return; } ImportGuard importGuard{_importStack, name}; - auto const &channelConstraints = _import->document.rooFit.channelConstraints; - auto found = channelConstraints.find(name); - HS3::buildHistFactoryDist(*this, name, std::get(distribution.body), - found != channelConstraints.end() ? &found->second : nullptr); + if (auto const *combined = std::get_if(&distribution.body)) { + HS3::buildCombinedDist(*this, name, *combined, otherLikelihoodTerms(_import->document, name)); + return; + } + HS3::buildHistFactoryDist(*this, name, std::get(distribution.body)); } -/** - * @brief Import a function from a JSON string into the workspace. - * - * This function imports a function from the provided JSON string into the workspace. - * The function's information is read from the JSON string and added to the workspace. - * - * @param jsonString The JSON string containing the function information. - * @param importAllDependants A boolean flag indicating whether to import all dependants (servers) of the function. - * @return void - */ +/// Import a function or distribution that is given as a JSON string, see the overload for a JSONNode. void RooJSONFactoryWSTool::importFunction(const std::string &jsonString, bool importAllDependants) { this->importFunction((JSONTree::create(jsonString))->rootnode(), importAllDependants); } -/** - * @brief Export the name and binning of a RooRealVar to a JSONNode. - * - * @param obsNode The JSONNode to which the axis information will be exported. - * @param var The RooRealVar representing the axis to be exported. - * @return void - */ -// A categorical axis: the labels in the order of the states of the category, which is the order of the bins of a -// RooDataHist. -void RooJSONFactoryWSTool::exportCategoricalAxis(JSONNode &obsNode, RooCategory const &cat) -{ - testValidName(cat.GetName(), false); - obsNode["name"] << cat.GetName(); - auto &labels = obsNode["categories"].set_seq(); - for (unsigned int i = 0; i < cat.size(); ++i) { - labels.append_child() << cat.getOrdinal(i).first; - } -} - -void RooJSONFactoryWSTool::exportAxis(JSONNode &obsNode, RooRealVar const &var) -{ - std::string name = var.GetName(); - RooJSONFactoryWSTool::testValidName(name, false); - obsNode["name"] << name; - - writeAxisBinning(obsNode, var.getBinning()); -} - -/** - * @brief Export histogram data to a JSONNode. - * - * This function exports histogram data, represented by the provided variables and contents, to a JSONNode. - * The histogram's axes information and bin contents are added as key-value pairs to the JSONNode. - * - * @param vars The RooArgSet representing the variables associated with the histogram. - * @param n The number of bins in the histogram. - * @param contents A pointer to the array containing the bin contents of the histogram. - * @param output The JSONNode to which the histogram data will be exported. - * @return void - */ +/// Write the axes and the `n` bin contents of a histogram over the variables `vars`, which are RooRealVars or +/// RooCategories, in the order of the variables. void RooJSONFactoryWSTool::exportHisto(RooArgSet const &vars, std::size_t n, double const *contents, JSONNode &output) { - auto &observablesNode = output["axes"].set_seq(); - // axes have to be ordered to get consistent bin indices - for (RooAbsArg const *var : vars) { - if (auto const *cat = dynamic_cast(var)) { - exportCategoricalAxis(observablesNode.append_child().set_map(), *cat); - } else { - exportAxis(observablesNode.append_child().set_map(), static_cast(*var)); - } - } - - return exportArray(n, contents, output["contents"]); + HS3::writeBinnedData(output, toBinnedData(vars, n, contents)); } -/** - * @brief Export an array of doubles to a JSONNode. - * - * This function exports an array of doubles, represented by the provided size and contents, - * to a JSONNode. The array elements are added to the JSONNode as a sequence of values. - * - * @param n The size of the array. - * @param contents A pointer to the array containing the double values. - * @param output The JSONNode to which the array will be exported. - * @return void - */ +/// Write `n` numbers as an array, each as an integer if it has no fractional part. RooFit doesn't use this function +/// itself, but exporters outside of ROOT do. void RooJSONFactoryWSTool::exportArray(std::size_t n, double const *contents, JSONNode &output) { - output.set_seq(); - for (std::size_t i = 0; i < n; ++i) { - double w = contents[i]; - // To make sure there are no unnecessary floating points in the JSON - if (int(w) == w) { - output.append_child() << int(w); - } else { - output.append_child() << w; - } - } + HS3::writeContents(output, std::vector(contents, contents + n)); } // Config::renameCombinedObservables: work out, before anything is exported, the new names of the observables of @@ -2842,7 +3049,7 @@ void RooJSONFactoryWSTool::prepareCombinedObservableRenaming() if (columns.empty()) { continue; } - RooSimultaneous const *simPdf = findMatchingSimultaneous(_workspace, selectionColumnNames(*data->get())); + RooSimultaneous const *simPdf = findMatchingSimultaneous(_workspace, namesOf(selectionColumns(*data->get()))); for (std::vector const &key : selectionKeys(columns)) { const std::string keyName = selectionKeyName(key); RooArgSet candidates; @@ -2911,18 +3118,18 @@ void RooJSONFactoryWSTool::prepareCombinedObservableRenaming() * the observables of the corresponding component, and keys without entries for which it has no component are left * out; otherwise, every sub-data set carries all observables. */ -RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooAbsData const &data) +void RooJSONFactoryWSTool::exportCombinedData(RooAbsData const &data) { std::vector columns = selectionColumns(*data.get()); - CombinedData info; - info.name = data.GetName(); + std::string const name = data.GetName(); + HS3::CombinedData combined; for (RooCategory const *column : columns) { - info.columns.push_back(column->GetName()); + combined.columns.push_back(column->GetName()); recordCategory(*column); } // Split the data by the key: by the index category of a matching RooSimultaneous, or by the columns themselves. - RooSimultaneous const *simPdf = findMatchingSimultaneous(_workspace, info.columns); + RooSimultaneous const *simPdf = findMatchingSimultaneous(_workspace, combined.columns); std::unique_ptr superCat; RooAbsCategory const *indexCat = nullptr; std::vector> parts; @@ -2937,15 +3144,13 @@ RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooA for (RooCategory const *column : columns) { inputs.add(*column); } - const std::string superName = info.name + "_key"; + const std::string superName = name + "_key"; superCat = std::make_unique(superName.c_str(), superName.c_str(), inputs); indexCat = superCat.get(); parts = data.split(*indexCat, true); } // The combined data set is written before its sub-data sets. - HS3::CombinedData combined; - combined.columns = info.columns; std::vector subDataSets; for (std::vector const &key : selectionKeys(columns)) { std::string const label = selectionLabel(*indexCat, columns, key); @@ -2954,10 +3159,10 @@ RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooA continue; } const std::string keyName = selectionKeyName(key); - const std::string subName = info.name + "_" + keyName; + const std::string subName = name + "_" + keyName; (*found)->SetName(subName.c_str()); // Config::renameCombinedObservables: give the observables of this key their own names. - auto renamesOfData = _combinedObservableRenames.find(info.name); + auto renamesOfData = _combinedObservableRenames.find(name); if (renamesOfData != _combinedObservableRenames.end()) { auto renames = renamesOfData->second.find(keyName); if (renames != renamesOfData->second.end()) { @@ -2969,11 +3174,10 @@ RooJSONFactoryWSTool::CombinedData RooJSONFactoryWSTool::exportCombinedData(RooA combined.components.push_back({key, {subName}}); subDataSets.push_back(found->get()); } - _export->doc.data.push_back({info.name, std::move(combined)}); + _export->doc.data.push_back({name, std::move(combined)}); for (RooAbsData *subData : subDataSets) { exportData(*subData); } - return info; } // Export a single dataset `data` (binned or unbinned) to the output document. @@ -2986,8 +3190,7 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) } } - HS3::Data output; - output.name = data.GetName(); + auto write = [&](auto body) { _export->doc.data.push_back({data.GetName(), std::move(body)}); }; // This works around a problem in RooStats/HistFactory that was only fixed // in ROOT 6.30: until then, the weight variable of the observed dataset, @@ -3000,37 +3203,19 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) variables.remove(*weightVar); } - // Categories are categorical axes, declared by a categorical axis of the default domain. - auto categoricalAxis = [&](RooCategory const &cat) { - recordCategory(cat); - HS3::CategoricalAxis axis; - axis.name = cat.GetName(); - testValidName(axis.name, false); - for (unsigned int i = 0; i < cat.size(); ++i) { - axis.categories.push_back(cat.getOrdinal(i).first); - } - return axis; - }; - - auto exportBinned = [&](std::size_t n, double const *contents) { - HS3::BinnedData binned; - for (RooAbsArg const *arg : variables) { - if (auto const *cat = dynamic_cast(arg)) { - binned.axes.emplace_back(categoricalAxis(*cat)); - continue; - } - auto const *var = static_cast(arg); - recordDomains(_export->domains, *var); - binned.axes.emplace_back(HS3::toBinnedAxis(*var)); + // The ranges of the variables are written to the domains. Categories are declared by a categorical axis of the + // default domain. + for (RooAbsArg const *arg : variables) { + if (auto const *cat = dynamic_cast(arg)) { + recordCategory(*cat); + } else { + recordDomains(_export->domains, static_cast(*arg)); } - binned.contents.assign(contents, contents + n); - output.body = std::move(binned); - _export->doc.data.push_back(std::move(output)); - }; + } // this is a regular binned dataset if (auto dh = dynamic_cast(&data)) { - return exportBinned(dh->numEntries(), dh->weightArray()); + return write(toBinnedData(variables, dh->numEntries(), dh->weightArray())); } // Check if this actually represents a binned dataset, and then import it @@ -3042,7 +3227,6 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) // binning, and would otherwise be written as binned once they have no entries. if (data.isWeighted() && variables.size() == 1 && dynamic_cast(variables[0]) && static_cast(*variables[0]).getBins() > 0) { - bool isBinnedData = false; auto &x = static_cast(*variables[0]); std::vector contents; int i = 0; @@ -3054,28 +3238,36 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) } // Binned data has exactly one entry per bin, in the order of the bins. Further entries, e.g. a second one in // the last bin, make it unbinned data. - if (i == x.getBins() && data.numEntries() == x.getBins()) - isBinnedData = true; - if (isBinnedData) { - return exportBinned(contents.size(), contents.data()); + if (i == x.getBins() && data.numEntries() == x.getBins()) { + return write(toBinnedData(variables, contents.size(), contents.data())); + } + } + + // A single unweighted entry of a single variable is point data. + if (variables.size() == 1 && data.numEntries() == 1) { + data.get(0); + auto const *var = dynamic_cast(variables[0]); + if (var && (!data.isWeighted() || data.weight() == 1.)) { + HS3::PointData point; + point.value = var->getVal(); + point.axes.push_back(toContinuousAxis(*var)); + return write(std::move(point)); } } // this really is an unbinned dataset HS3::UnbinnedData unbinned; for (RooAbsArg const *arg : variables) { + testValidName(arg->GetName(), false); if (auto const *cat = dynamic_cast(arg)) { - unbinned.axes.emplace_back(categoricalAxis(*cat)); - continue; + unbinned.axes.emplace_back(toCategoricalAxis(*cat)); + } else { + // HS3 forbids binning information on the axes of unbinned data. + auto const &var = static_cast(*arg); + unbinned.axes.emplace_back(toContinuousAxis(var)); } - auto const *var = static_cast(arg); - recordDomains(_export->domains, *var); - // HS3 forbids binning information on the axes of unbinned data. - testValidName(var->GetName(), false); - unbinned.axes.emplace_back(HS3::ContinuousAxis{var->GetName(), var->getMin(), var->getMax()}); } - std::vector weightVals; - bool hasNonUnityWeights = false; + std::vector weights; for (int i = 0; i < data.numEntries(); ++i) { data.get(i); // The entries of a categorical axis are the positions of the labels in its "categories". @@ -3086,28 +3278,16 @@ void RooJSONFactoryWSTool::exportData(RooAbsData const &data) unbinned.entries.push_back(static_cast(arg)->getVal()); } } - if (data.isWeighted()) { - weightVals.push_back(data.weight()); - if (data.weight() != 1.) - hasNonUnityWeights = true; - } + weights.push_back(data.weight()); } - if (data.isWeighted() && hasNonUnityWeights) { - unbinned.weights = std::move(weightVals); + // Weights that are all one are not written. + if (std::any_of(weights.begin(), weights.end(), [](double weight) { return weight != 1.; })) { + unbinned.weights = std::move(weights); } - output.body = std::move(unbinned); - _export->doc.data.push_back(std::move(output)); + write(std::move(unbinned)); } -/** - * @brief Read axes from the JSONNode and create a RooArgSet representing them. - * - * This function reads axes information from the given JSONNode and - * creates a RooArgSet with variables representing these axes. - * - * @param topNode The JSONNode containing the axes information to be read. - * @return RooArgSet A RooArgSet containing the variables created from the JSONNode. - */ +/// Create the variables for the "axes" of `topNode`, which are owned by the returned set. RooArgSet RooJSONFactoryWSTool::readAxes(const JSONNode &topNode) { HS3::Diagnostics diagnostics; @@ -3116,17 +3296,8 @@ RooArgSet RooJSONFactoryWSTool::readAxes(const JSONNode &topNode) return HS3::createObservables(axes); } -/** - * @brief Read binned data from the JSONNode and create a RooDataHist object. - * - * This function reads binned data from the given JSONNode and creates a RooDataHist object. - * The binned data is associated with the specified name and variables (RooArgSet) in the workspace. - * - * @param n The JSONNode representing the binned data to be read. - * @param name The name to be associated with the created RooDataHist object. - * @param vars The RooArgSet representing the variables associated with the binned data. - * @return std::unique_ptr A unique pointer to the created RooDataHist object. - */ +/// Create a RooDataHist with the name `name` over the variables `vars` from the "contents" of `n`, and its "errors" +/// if it has any. std::unique_ptr RooJSONFactoryWSTool::readBinnedData(const JSONNode &n, const std::string &name, RooArgSet const &vars) { @@ -3154,7 +3325,6 @@ RooJSONFactoryWSTool::readBinnedData(const JSONNode &n, const std::string &name, // Import a single variable (RooRealVar or RooConstVar) from the parameter `p` into the workspace. void RooJSONFactoryWSTool::importVariable(HS3::ParameterValue const &p) { - // import a RooRealVar object RooJSONFactoryWSTool::testValidName(p.name, true); if (_workspace.arg(p.name)) @@ -3201,49 +3371,64 @@ void RooJSONFactoryWSTool::importDependants(const JSONNode &n) } } +// Export a ModelConfig as one analysis for every data set that goes with its pdf, or as one without data. void RooJSONFactoryWSTool::exportModelConfig(RooFit::ModelConfig const &mc, - const std::vector &combDataSets, const std::vector &singleDataSets) { - // The analyses are named after the ModelConfig, so that ModelConfigs sharing a pdf and a data set (like an S+B - // and B-only pair) get separate analyses. - auto pdf = mc.GetPdf(); - auto simpdf = dynamic_cast(pdf); - int founddata = 0; - if (simpdf) { - // One analysis per combined data set with the same columns. - std::vector columns; - for (RooCategory const *column : selectionColumns(simpdf->indexCat())) { - columns.push_back(column->GetName()); - } - for (auto const &data : combDataSets) { - if (data.columns == columns) { - exportSingleModelConfig(mc, std::string(mc.GetName()) + "_" + data.name, &data.name); - ++founddata; + std::vector dataNames; + if (auto simpdf = dynamic_cast(mc.GetPdf())) { + // The combined data sets with the same columns. + const std::vector columns = namesOf(selectionColumns(simpdf->indexCat())); + for (HS3::Data const &data : _export->doc.data) { + auto const *combined = std::get_if(&data.body); + if (combined && combined->columns == columns) { + dataNames.push_back(data.name); } } } else { - RooArgSet observables(*mc.GetObservables()); - for (auto *data : singleDataSets) { - if (observables.equals(*(data->get()))) { - const std::string dataName = data->GetName(); - exportSingleModelConfig(mc, std::string(mc.GetName()) + "_" + dataName, &dataName); - ++founddata; + for (RooAbsData const *data : singleDataSets) { + if (mc.GetObservables()->equals(*data->get())) { + dataNames.push_back(data->GetName()); } } } - if (founddata == 0) { + // The analyses are named after the ModelConfig, so that ModelConfigs sharing a pdf and a data set (like an S+B + // and B-only pair) get separate analyses. + for (std::string const &dataName : dataNames) { + exportSingleModelConfig(mc, std::string(mc.GetName()) + "_" + dataName, &dataName); + } + if (dataNames.empty()) { exportSingleModelConfig(mc, mc.GetName(), nullptr); } } namespace { +// The channels of the pdf of a ModelConfig as they are written: the pdf itself, or the components of a RooSimultaneous, +// of which those in `renamed` (by label) replace the original ones (Config::renameCombinedObservables). +std::vector channelsOf(RooAbsPdf const &pdf, std::map const &renamed) +{ + auto const *simPdf = dynamic_cast(&pdf); + if (!simPdf) { + return {&pdf}; + } + std::vector channels; + for (auto const &item : simPdf->indexCat()) { + auto found = renamed.find(item.first); + if (RooAbsPdf const *component = found != renamed.end() ? found->second : simPdf->getPdf(item.first)) { + channels.push_back(component); + } + } + return channels; +} + // A constraint term of a channel of the pdf of a ModelConfig, see constraintsOfChannels(). struct ChannelConstraint { std::string channel; RooAbsPdf const *constraint = nullptr; bool histFactory = false; ///< Whether the channel is written as a histfactory_dist. + /// Whether the channel consists of constraint terms only; `constraint` is then the channel itself. + bool wholeChannel = false; }; // The constraint terms of the pdf `pdf` of a ModelConfig with the observables `observables`, which are written as @@ -3257,33 +3442,16 @@ struct ChannelConstraint { // - The constraints of a channel that is written as product_dist are taken out of the product by // writeLikelihoodTermsOnce(). This is only done if no other object of the workspace uses the product, so the // constraints of other products are not returned. -// Constraints elsewhere, e.g. in a nested product, stay where they are. The channels are taken as they are written: -// a component of the RooSimultaneous is replaced by the one in `renamed` (by label), if any -// (Config::renameCombinedObservables). +// Constraints elsewhere, e.g. in a nested product, stay where they are. The channels are taken as they are written, +// see channelsOf(). std::vector constraintsOfChannels(RooWorkspace const &ws, RooAbsPdf const &pdf, RooArgSet const &observables, std::function const &isHistFactoryChannel, - std::map const *renamed) + std::vector const &channels) { RooArgSet params; pdf.getParameters(&observables, params); std::unique_ptr constraints{pdf.getAllConstraints(observables, params, /*stripDisconnected=*/false)}; - std::vector channels; - if (auto const *simPdf = dynamic_cast(&pdf)) { - for (auto const &item : simPdf->indexCat()) { - RooAbsPdf const *component = simPdf->getPdf(item.first); - if (renamed) { - if (auto found = renamed->find(item.first); found != renamed->end()) { - component = found->second; - } - } - if (component) { - channels.push_back(component); - } - } - } else { - channels.push_back(&pdf); - } std::vector out; for (RooAbsPdf const *channel : channels) { auto const *prod = dynamic_cast(channel); @@ -3291,6 +3459,14 @@ constraintsOfChannels(RooWorkspace const &ws, RooAbsPdf const &pdf, RooArgSet co continue; } const bool histFactory = isHistFactoryChannel(prod->GetName()); + // A channel that consists of constraint terms only is an auxiliary measurement that is part of the model. It + // is written as that channel, once, and not as constraint terms of the likelihood. + if (!histFactory && !prod->pdfList().empty() && + std::all_of(prod->pdfList().begin(), prod->pdfList().end(), + [&](RooAbsArg const *factor) { return constraints->find(*factor) != nullptr; })) { + out.push_back({prod->GetName(), prod, false, true}); + continue; + } if (!histFactory) { // Being a component of a RooSimultaneous, i.e. of a combined distribution, is not a use. bool usedElsewhere = false; @@ -3341,48 +3517,36 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc if (dataName) { likelihood.terms.push_back({{pdf->GetName()}, HS3::DataRef{*dataName}}); } else { - likelihood.terms.push_back({{pdf->GetName()}, 0.0}); + // Without data, the likelihood is a pure model, which is written with "data": null. + likelihood.terms.push_back({{pdf->GetName()}, std::monostate{}}); + likelihood.pureModel = true; } - auto addAux = [&](std::string const &name) { - auto &aux = likelihood.auxDistributions; - if (std::find_if(aux.begin(), aux.end(), [&](auto const &ref) { return ref.name == name; }) == aux.end()) { - aux.push_back({name}); - } - }; // The additional terms of the likelihood: the external constraints of the ModelConfig, and the constraint terms // of the channels of the model, which can be the components of a combined distribution (constraintsOfChannels()). // They are listed in "aux_distributions" with the observed values of their observables as constant parameters, or // (config().exportConstraintData) in "distributions", each paired with a data set that holds the observed values. - // The channels as they are written: the components of a RooSimultaneous can be renamed clones - // (Config::renameCombinedObservables). - std::vector channels; - for (auto const &term : likelihood.terms) { - channels.push_back(term.distribution.name); - if (auto const *simPdf = dynamic_cast(_workspace.pdf(term.distribution.name))) { - auto renamed = _renamedComponents.find(simPdf->GetName()); - for (auto const &item : simPdf->indexCat()) { - RooAbsPdf const *component = simPdf->getPdf(item.first); - if (renamed != _renamedComponents.end()) { - if (auto found = renamed->second.find(item.first); found != renamed->second.end()) { - component = found->second; - } - } - if (component) { - channels.push_back(component->GetName()); + std::vector const channels = channelsOf(*pdf, _renamedComponents[pdf->GetName()]); + // The channels of the model that consist of constraint terms only, see constraintsOfChannels(). + std::vector auxiliaryChannelsOfModel; + std::vector constraintTerms; + if (RooArgSet const *external = mc.GetExternalConstraints()) { + // A component of an array distribution is written as part of the array, which enters the likelihood as a + // whole: the external constraints must contain all of its components. + for (const auto &constr : *external) { + auto array = _export->arrayOfComponent.find(constr->GetName()); + if (array == _export->arrayOfComponent.end()) { + addUnique(constraintTerms, constr->GetName()); + continue; + } + for (RooAbsPdf const *component : _export->arrayComponents[array->second]) { + if (!external->find(*component)) { + error("cannot write the external constraint '" + std::string(constr->GetName()) + "' of '" + + mc.GetName() + "': it is written as a component of the array distribution '" + array->second + + "', whose component '" + component->GetName() + "' is not an external constraint"); } } - } - } - std::vector constraintTerms; - auto addConstraintTerm = [&](std::string const &term) { - if (std::find(constraintTerms.begin(), constraintTerms.end(), term) == constraintTerms.end()) { - constraintTerms.push_back(term); - } - }; - if (mc.GetExternalConstraints()) { - for (const auto &constr : *mc.GetExternalConstraints()) { - addConstraintTerm(constr->GetName()); + addUnique(constraintTerms, array->second); } } // The constraint terms that the HistFactory exporter wrote for the channel `channel`, in the order of the document: @@ -3390,19 +3554,14 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc // names. auto writtenByHistFactory = [&](std::string const &channel) { std::vector out; - auto add = [&](std::string const &name) { - if (std::find(out.begin(), out.end(), name) == out.end()) { - out.push_back(name); - } - }; if (auto found = _export->constraintTermsOfPdf.find(channel); found != _export->constraintTermsOfPdf.end()) { for (std::string const &term : found->second) { - add(term); + addUnique(out, term); } } if (auto found = doc.rooFit.channelConstraints.find(channel); found != doc.rooFit.channelConstraints.end()) { for (auto const &ref : found->second) { - add(ref.name); + addUnique(out, ref.name); } } return out; @@ -3411,41 +3570,37 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc if (!observables || observables->empty()) { // Without observables (e.g. a ModelConfig without data), RooFit can't tell constraints from other pdfs: only the // constraint terms of the HistFactory channels are listed. - for (std::string const &channel : channels) { - for (std::string const &term : writtenByHistFactory(channel)) { - addConstraintTerm(term); + for (RooAbsPdf const *channel : channels) { + for (std::string const &term : writtenByHistFactory(channel->GetName())) { + addUnique(constraintTerms, term); } } } else { auto isHistFactoryChannel = [&](std::string const &name) { - return _export->constraintTermsOfPdf.count(name) || - std::any_of(doc.distributions.begin(), doc.distributions.end(), [&](HS3::Distribution const &dist) { - return dist.name == name && std::holds_alternative(dist.body); - }); + return _export->constraintTermsOfPdf.count(name) || HS3::find(doc.distributions, name); }; // The constraint terms of the channels as RooFit finds them, under the names under which they are written: // the array distribution that holds a constraint of a HistFactory channel, or the name of the constraint. std::map> termsOfChannel; - auto renamedOfPdf = _renamedComponents.find(pdf->GetName()); for (ChannelConstraint const &item : - constraintsOfChannels(_workspace, *pdf, *observables, isHistFactoryChannel, - renamedOfPdf != _renamedComponents.end() ? &renamedOfPdf->second : nullptr)) { - std::string term = item.constraint->GetName(); - if (item.histFactory) { - if (auto found = _export->arrayOfComponent.find(term); found != _export->arrayOfComponent.end()) { - term = found->second; - } + constraintsOfChannels(_workspace, *pdf, *observables, isHistFactoryChannel, channels)) { + if (item.wholeChannel) { + auxiliaryChannelsOfModel.push_back(item.channel); + continue; } - auto &terms = termsOfChannel[item.channel]; - if (std::find(terms.begin(), terms.end(), term) == terms.end()) { - terms.push_back(term); + // A component of an array distribution is never referred to by itself: the array is the term. + std::string term = item.constraint->GetName(); + if (auto found = _export->arrayOfComponent.find(term); found != _export->arrayOfComponent.end()) { + term = found->second; } + addUnique(termsOfChannel[item.channel], term); } - for (std::string const &channel : channels) { + for (RooAbsPdf const *channelPdf : channels) { + std::string const channel = channelPdf->GetName(); std::vector const &found = termsOfChannel[channel]; if (!isHistFactoryChannel(channel)) { for (std::string const &term : found) { - addConstraintTerm(term); + addUnique(constraintTerms, term); } continue; } @@ -3467,24 +3622,112 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc "': the channel writes " + join(writtenSet) + ", but RooFit finds " + join(foundSet)); } for (std::string const &term : written) { - addConstraintTerm(term); + addUnique(constraintTerms, term); + } + } + } + // An array of constraints with its data set is a measurement like the others: if the model is a combined + // distribution, it is written as a component of it, paired with its data set as a component of the combined data + // set. The combined distribution is then the sampling distribution of all measurements, auxiliary or not. Its + // label is the name of the array. Constraints that are written one by one stay terms of the likelihood: there + // can be hundreds, which is not what an index category is for. So do the constraints of models whose index has + // several columns. + auto const *simPdf = dynamic_cast(pdf); + // Looked up when needed: writing the data set of a constraint term adds to the data sets of the document. + auto findCombinedData = [&]() { return dataName ? HS3::find(doc.data, *dataName) : nullptr; }; + RooCategory const *indexColumn = nullptr; + if (simPdf && findCombinedData()) { + std::vector columns = selectionColumns(simPdf->indexCat()); + if (columns.size() == 1) { + indexColumn = columns.front(); + } + } + auto addAuxiliaryChannel = [&](std::string const &term) { + if (indexColumn->hasLabel(term)) { + error("cannot write the constraint term '" + term + "' as a component of the combined distribution '" + + simPdf->GetName() + "': its index category '" + indexColumn->GetName() + "' has a state of that name"); + } + addUnique(_export->auxiliaryLabels[indexColumn->GetName()], term); + auto *combinedDist = HS3::find(doc.distributions, simPdf->GetName()); + if (!combinedDist) { + error("cannot write the constraint terms of '" + std::string(simPdf->GetName()) + + "' as its components: it was not written as a combined distribution"); + } + if (!combinedDist->find({term})) { + combinedDist->components.push_back({{term}, {term}}); + doc.rooFit.auxiliaryChannels[simPdf->GetName()].push_back({term}); + } + if (HS3::CombinedData *combinedData = findCombinedData(); !combinedData->find({term})) { + combinedData->components.push_back({{term}, HS3::DataRef{*dataName + "_" + term}}); + } + }; + // A channel of the model that consists of constraint terms is an auxiliary measurement, whose observed values are + // the values of its global observables if the data set has no entries for it. It is written like the arrays + // above: with a data set of the observed values, and marked for RooFit, which can read it as constraint terms. + for (std::string const &channel : auxiliaryChannelsOfModel) { + if (!indexColumn || likelihood.pureModel) { + continue; + } + std::string label; + for (auto const &state : simPdf->indexCat()) { + RooAbsPdf const *component = simPdf->getPdf(state.first); + if (component && channel == component->GetName()) { + label = state.first; + } + } + if (label.empty()) { + continue; + } + auto const *component = findCombinedData()->find({label}); + const std::string observed = component ? component->ref.name : *dataName + "_" + label; + // A data set without entries for the channel is replaced by the observed values. + auto written = std::find_if(doc.data.begin(), doc.data.end(), [&](auto const &d) { return d.name == observed; }); + if (written == doc.data.end() || written->numEntries() == 0) { + std::optional empty; + if (written != doc.data.end()) { + empty = std::move(*written); + doc.data.erase(written); + } + if (exportConstraintData(channel, mc.GetGlobalObservables(), observed)) { + if (HS3::CombinedData *combinedData = findCombinedData(); !combinedData->find({label})) { + combinedData->components.push_back({{label}, HS3::DataRef{observed}}); + } + } else { + if (empty) { + doc.data.push_back(std::move(*empty)); + } + warning("RooFitHS3: the channel '" + channel + "' of '" + simPdf->GetName() + + "' consists of constraint terms, but has no data, and its observables are not all global " + "observables of the ModelConfig: it is written without an observation."); + continue; } } + auto &marked = doc.rooFit.auxiliaryChannels[simPdf->GetName()]; + if (!HS3::findByName(marked, channel)) { + marked.push_back({channel}); + } } for (std::string const &term : constraintTerms) { - if (config().exportConstraintData && exportConstraintData(term, mc.GetGlobalObservables())) { - likelihood.terms.push_back({{term}, HS3::DataRef{term + "_data"}}); + // A pure model has no data at all, so the observed values of its constraint terms are not written as data sets + // either: they are the constant parameters of auxiliary distributions. + // A component of a combined data set belongs to that data set alone, and is named after it like the others. + const bool asChannel = indexColumn && _export->arrayComponents.count(term); + const std::string constraintData = asChannel ? *dataName + "_" + term : term + "_data"; + if (!likelihood.pureModel && config().exportConstraintData && + exportConstraintData(term, mc.GetGlobalObservables(), constraintData)) { + if (asChannel) { + addAuxiliaryChannel(term); + } else { + likelihood.terms.push_back({{term}, HS3::DataRef{constraintData}}); + } } else { - addAux(term); + likelihood.auxDistributions.push_back({term}); } } doc.likelihoods.push_back(std::move(likelihood)); if (RooArgSet const *pois = mc.GetParametersOfInterest()) { - std::vector names; - names.reserve(pois->size()); - for (RooAbsArg const *arg : *pois) - names.push_back(arg->GetName()); + std::vector names = namesOf(*pois); std::sort(names.begin(), names.end()); for (auto const &name : names) { analysis.parametersOfInterest.push_back({name}); @@ -3523,10 +3766,7 @@ void RooJSONFactoryWSTool::exportSingleModelConfig(RooFit::ModelConfig const &mc info.mcName = mc.GetName(); // Without a data set, the document doesn't say what the observables are. if (!dataName && mc.GetObservables() && !mc.GetObservables()->empty()) { - std::vector names; - for (RooAbsArg const *arg : *mc.GetObservables()) { - names.push_back(arg->GetName()); - } + std::vector names = namesOf(*mc.GetObservables()); std::sort(names.begin(), names.end()); info.observables = std::move(names); } @@ -3537,11 +3777,7 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) { _export = std::make_unique(); _export->root = &n; - // The state of the export must not survive it, not even if there is an exception. - struct ExportScope { - RooJSONFactoryWSTool &tool; - ~ExportScope() { tool._export.reset(); } - } exportScope{*this}; + StateScope exportScope{_export}; HS3Document &doc = _export->doc; doc.metadata.hs3Version = hs3VersionTag; @@ -3549,39 +3785,34 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) doc.parameterPoints.push_back({"default_values", {}}); prepareCombinedObservableRenaming(); - // export all toplevel pdfs - std::vector allpdfs; - for (auto &arg : _workspace.allPdfs()) { - if (isTopLevel(*arg, _workspace)) { - if (auto *pdf = dynamic_cast(arg)) { - allpdfs.push_back(pdf); - } + std::vector const allpdfs = topLevelObjects(_workspace.allPdfs(), _workspace); + // A Gaussian constraint is written with its global observable as "x", also if that is its mean in RooFit (see + // setConstraintObservable()). Which one it is must be known before any pdf is written: the pdfs are written in the + // order of their names, and a constraint can come before the model that uses it. + for (TObject *obj : _workspace.allGenericObjects()) { + auto const *mc = dynamic_cast(obj); + RooArgSet const *globs = mc ? mc->GetGlobalObservables() : nullptr; + if (!globs) { + continue; } - } - sortByName(allpdfs); - exportObjects(allpdfs); - - // export all toplevel functions - std::vector allfuncs; - for (auto &arg : _workspace.allFunctions()) { - if (isTopLevel(*arg, _workspace)) { - if (auto *func = dynamic_cast(arg)) { - allfuncs.push_back(func); + for (RooAbsArg const *arg : _workspace.components()) { + auto const *gauss = dynamic_cast(arg); + if (gauss && globs->find(gauss->getMean()) && !globs->find(gauss->getX()) && + !constraintObservable(gauss->GetName())) { + setConstraintObservable(*gauss, gauss->getMean().GetName()); } } } - sortByName(allfuncs); - exportObjects(allfuncs); + exportObjects(allpdfs); + exportObjects(topLevelObjects(_workspace.allFunctions(), _workspace)); // The array distributions are written after the objects that use them, see exportAsArrayComponent(). exportArrayDists(); - // export attributes of all objects for (RooAbsArg *arg : _workspace.components()) { exportAttributes(arg, doc.rooFit.attributes); } - // collect all datasets std::vector alldata; for (auto &d : _workspace.allData()) { alldata.push_back(d); @@ -3589,24 +3820,22 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) sortByName(alldata); // Data sets with category observables are combined data sets. std::vector singleData; - std::vector combData; for (auto &d : alldata) { // Without a matching RooSimultaneous, the categories of a data set are ordinary observables, written as // categorical axes, unless its observables are renamed per key (Config::renameCombinedObservables). if (selectionColumns(*d->get()).empty() || (!config().renameCombinedObservables && - !findMatchingSimultaneous(_workspace, selectionColumnNames(*d->get())))) { + !findMatchingSimultaneous(_workspace, namesOf(selectionColumns(*d->get()))))) { singleData.push_back(d); this->exportData(*d); } else { - combData.push_back(this->exportCombinedData(*d)); + this->exportCombinedData(*d); } } - // export all ModelConfig objects and attached Pdfs for (TObject *obj : _workspace.allGenericObjects()) { if (auto mc = dynamic_cast(obj)) { - exportModelConfig(*mc, combData, singleData); + exportModelConfig(*mc, singleData); } } @@ -3618,13 +3847,7 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) // the ones that the pdfs encoded implicitly (like in the case of // HistFactory). for (RooAbsArg *arg : *snsh) { - bool do_export = false; - for (const auto &pdf : allpdfs) { - if (pdf->dependsOn(*arg)) { - do_export = true; - } - } - if (do_export) { + if (std::any_of(allpdfs.begin(), allpdfs.end(), [&](RooAbsPdf *pdf) { return pdf->dependsOn(*arg); })) { RooJSONFactoryWSTool::testValidName(arg->GetName(), true); snapshotSorted.add(*arg); } @@ -3633,7 +3856,18 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) std::string name(snsh->GetName()); if (name != "default_values") { doc.parameterPoints.push_back({name, {}}); - this->exportVariables(snapshotSorted, doc.parameterPoints.back(), true); + for (RooAbsArg *arg : snapshotSorted) { + exportVariable(arg, doc.parameterPoints.back()); + } + } + } + // The constraint terms that are components of a combined distribution have a label of their own. + for (auto const &item : _export->auxiliaryLabels) { + auto &categories = _export->categories[item.first].categories; + for (std::string const &label : item.second) { + if (std::find(categories.begin(), categories.end(), label) == categories.end()) { + categories.push_back(label); + } } } // The categories are categorical axes of the "default_domain", which must not list them as continuous. @@ -3646,14 +3880,46 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) } // The domains of the analyses take precedence. for (auto &item : _export->domains) { - if (std::none_of(doc.domains.begin(), doc.domains.end(), - [&](HS3::Domain const &domain) { return domain.name == item.first; })) { + if (!HS3::findByName(doc.domains, item.first)) { appendDomain(doc.domains, item.first, std::move(item.second), item.first == HS3::defaultDomainName ? std::move(_export->categories) : std::map{}); } } + // The exporters have written the generic distributions to the JSON tree. Now that the tree has all of them, the + // document refers to their nodes, like a document that was read. The combined distributions are written now, since + // constraint terms can become their components until here, see exportSingleModelConfig(). + if (n.has_child("distributions")) { + std::map nodes; + for (JSONNode &node : n["distributions"].children()) { + nodes.emplace(name(node), &node); + } + for (HS3::Distribution &dist : doc.distributions) { + auto found = nodes.find(dist.name); + if (found == nodes.end()) { + continue; + } + if (auto *generic = std::get_if(&dist.body)) { + generic->node = found->second; + } else if (auto const *combined = std::get_if(&dist.body)) { + HS3::writeCombinedDist(*found->second, *combined); + } + } + } + + // A RooFit model often has a constraint both as a factor of the product of a channel and as a term of the + // likelihood (e.g. a constraint of a HistFactory modifier, an external constraint of the ModelConfig, or any + // constraint of a channel, see constraintsOfChannels()); RooFit counts it once. In HS3 it would enter twice. + for (auto const &[product, factors] : HS3::writeLikelihoodTermsOnce(doc, n)) { + for (std::string const &factor : factors) { + oocoutI(nullptr, IO) << "RooFitHS3: '" << factor << "' is a factor of '" << product + << "' and a term of its likelihood; it is written once, as a term of the likelihood" + << std::endl; + } + } + declareBookkeepingColumns(doc, _workspace); + // The document that RooFit writes has to pass the same checks as the ones that it reads. Until RooFit writes // documents that follow the current standard, problems are only reported. HS3::Diagnostics diagnostics; @@ -3668,12 +3934,9 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) HS3::report(diagnostics); HS3::writeDocument(n, doc); - // Each term of a likelihood once, and distinct variates (see sharedVariates()). Otherwise, the document is only - // written if explicitly allowed, e.g. to rename the variates in the JSON by hand. This is checked on the written - // document, which has the generic distributions. - writeLikelihoodTermsOnce(n); - declareBookkeepingColumns(n, _workspace); - if (std::vector const shared = sharedVariatesOfDocument(n); !shared.empty()) { + // Without distinct variates (see HS3::sharedVariates()), the document is only written if explicitly allowed, e.g. + // to rename the variates in the JSON by hand. + if (std::vector const shared = HS3::sharedVariates(doc); !shared.empty()) { if (!config().allowExportSharedVariates) { std::string text; for (std::string const &message : shared) { @@ -3694,23 +3957,14 @@ void RooJSONFactoryWSTool::exportAllObjects(JSONNode &n) } } -/** - * @brief Import the workspace from a JSON string. - * - * @param s The JSON string containing the workspace data. - * @return bool Returns true on successful import, false otherwise. - */ +/// Import an HS3 document that is given as a JSON string into the workspace. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::importJSONfromString(const std::string &s) { std::stringstream ss(s); return importJSON(ss); } -/** - * @brief Export the workspace to a JSON string. - * - * @return std::string The JSON string representing the exported workspace. - */ +/// Export the workspace as an HS3 document, and return it as a JSON string. std::string RooJSONFactoryWSTool::exportJSONtoString() { std::stringstream ss; @@ -3718,32 +3972,7 @@ std::string RooJSONFactoryWSTool::exportJSONtoString() return ss.str(); } -/** - * @brief Create a new JSON tree with version information. - * - * @return std::unique_ptr A unique pointer to the created JSON tree. - */ -std::unique_ptr RooJSONFactoryWSTool::createNewJSONTree() -{ - std::unique_ptr tree = JSONTree::create(); - JSONNode &n = tree->rootnode(); - n.set_map(); - auto &metadata = n["metadata"].set_map(); - - // add the mandatory hs3 version number - metadata["hs3_version"] << hs3VersionTag; - - appendNamedChild(metadata["packages"], "ROOT")["version"] << rootVersion(); - - return tree; -} - -/** - * @brief Export the workspace to JSON format and write to the output stream. - * - * @param os The output stream to write the JSON data to. - * @return bool Returns true on successful export, false otherwise. - */ +/// Export the workspace as an HS3 document, and write it as JSON to the stream. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::exportJSON(std::ostream &os) { std::unique_ptr tree = JSONTree::create(); @@ -3753,12 +3982,7 @@ bool RooJSONFactoryWSTool::exportJSON(std::ostream &os) return true; } -/** - * @brief Export the workspace to JSON format and write to the specified file. - * - * @param filename The name of the JSON file to create and write the data to. - * @return bool Returns true on successful export, false otherwise. - */ +/// Export the workspace as an HS3 document, and write it to a JSON file. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::exportJSON(std::string const &filename) { std::ofstream out(filename.c_str()); @@ -3816,25 +4040,25 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) std::optional validated; if (!HS3::hasErrors(diagnostics)) { HS3::ValidateOptions validateOptions; - validateOptions.strict = !isHS3VersionBefore(parsed.metadata.hs3Version, 0, 3); + validateOptions.strict = !(parsed.metadata.version < HS3::Version{0, 3}); validateOptions.isExternal = [this](std::string const &name) { return _workspace.obj(name) != nullptr; }; validated = HS3::validate(std::move(parsed), diagnostics, validateOptions); } HS3::report(diagnostics); + // What HS3 requires of the variates and the constraints is checked on the document as it is, before the components + // that are read as constraint terms are taken out of it. + std::set const bookkeepingColumns = checkVariates(validated->doc()); + HS3::AuxiliaryConstraints const auxiliary = + HS3::readAuxiliaryChannelsAsConstraints(*validated, config().importConstraintsAsChannels); + for (std::string const &message : auxiliary.warnings) { + warning("RooFitHS3: " + message); + } + for (std::string const &message : HS3::resolvePointData(*validated)) { + error(message); + } HS3Document const &doc = validated->doc(); - // The state of the import refers to the document, so it must not survive it, not even if there is an exception. - struct ImportScope { - RooJSONFactoryWSTool &tool; - ~ImportScope() - { - tool._import.reset(); - tool._hs3VersionInput.clear(); - } - } importScope{*this}; - - _hs3VersionInput = doc.metadata.hs3Version; - + StateScope importScope{_import}; _import = std::make_unique(doc); createVariables(_workspace, _import->document.domains, _import->ranges); @@ -3892,7 +4116,6 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) _workspace.saveSnapshot(point.name, vars); } - // Import attributes for (auto const &item : doc.rooFit.attributes) { if (RooAbsArg *arg = _workspace.arg(item.first)) importAttributes(arg, item.second); @@ -3904,59 +4127,25 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) std::vector> datasets; datasets.reserve(doc.data.size()); for (auto const &data : doc.data) { - // Combined data sets are assembled from the others once all of them are loaded. if (!std::holds_alternative(data.body)) { datasets.push_back(loadData(data, _workspace, &_import->ranges)); } } assembleCombinedData(doc, _workspace, datasets); - // Now, read in analyses and likelihoods if there are any - - // Distinct variates (see sharedVariates()). Documents of versions before 0.3, which RooFit wrote with shared - // variates itself, are read with a warning. - std::set bookkeepingColumns; - for (auto const &[dataset, columns] : bookkeepingColumnsOfDocument(n)) { - for (std::string const &column : columns) { - warning("RooFitHS3: the axis '" + column + "' of the components of the combined data set '" + dataset + - "' is declared as a bookkeeping column without statistical meaning (misc.ROOT_internal." - "bookkeeping_columns), so it may be shared by the components, although HS3 requires distinct axes."); - bookkeepingColumns.insert(column); - } - } - for (std::string const &message : sharedVariatesOfDocument(n)) { - if (!isHS3VersionBefore(_hs3VersionInput, 0, 3)) { - error("invalid HS3: " + message + - ". The terms of a likelihood and the components of a combined distribution or data set must have " - "distinct variates."); - } - warning("RooFitHS3: " + message + - ". HS3 0.3 requires distinct variates; the document is read anyway, since it declares an older " - "version."); - } - for (auto const &analysis : doc.analyses) { - // The variates of the distributions of the likelihood, see observablesWithoutData(). - std::set variates; - for (auto const &likelihood : doc.likelihoods) { - if (likelihood.name == analysis.likelihood.name) { - for (auto const &term : likelihood.terms) { - std::set const v = variatesOf(n, term.distribution.name); - variates.insert(v.begin(), v.end()); - // The columns of a combined distribution are observables as well. - JSONNode const *dist = findInSection(n, "distributions", term.distribution.name); - if (dist && typeOf(*dist) == "combined_dist") { - if (JSONNode const *columns = dist->find("columns")) { - addNames(*columns, variates); - } - } - } - } - } - importAnalysis(doc, analysis, _workspace, datasets, bookkeepingColumns, variates); + importAnalysis(doc, analysis, _workspace, datasets, bookkeepingColumns, auxiliary.terms); } combineDatasets(doc, datasets); + // The data sets of the components that are read as constraint terms are not data sets of the workspace: their + // values are in the global observables. Several analyses can share a ModelConfig, of which only the first one + // consumes the data of its constraint terms. + for (auto &d : datasets) { + if (d && auxiliary.data.count(d->GetName())) { + d.reset(); + } + } for (auto const &d : datasets) { if (d) { @@ -3977,29 +4166,17 @@ void RooJSONFactoryWSTool::importAllNodes(JSONNode &n) importParameterStepWidths(_workspace, doc); } -/** - * @brief Imports a JSON file from the given input stream to the workspace. - * - * @param is The input stream containing the JSON data. - * @return bool Returns true on successful import, false otherwise. - */ +/// Import an HS3 document that is read as JSON from the stream into the workspace. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::importJSON(std::istream &is) { - // import a JSON file to the workspace std::unique_ptr tree = JSONTree::create(is); this->importAllNodes(tree->rootnode()); return true; } -/** - * @brief Imports a JSON file from the given filename to the workspace. - * - * @param filename The name of the JSON file to import. - * @return bool Returns true on successful import, false otherwise. - */ +/// Import the HS3 document of a JSON file into the workspace. Returns true; errors are exceptions. bool RooJSONFactoryWSTool::importJSON(std::string const &filename) { - // import a JSON file to the workspace std::ifstream infile(filename.c_str()); if (!infile.is_open()) RooJSONFactoryWSTool::error("RooJSONFactoryWSTool() invalid input file '" + filename + "'."); @@ -4008,7 +4185,6 @@ bool RooJSONFactoryWSTool::importJSON(std::string const &filename) void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std::string &jsonString) { - // Create the JSON Tree from the string std::unique_ptr tree = RooFit::Detail::JSONTree::create(jsonString); JSONNode &n = tree->rootnode(); @@ -4028,24 +4204,13 @@ void RooJSONFactoryWSTool::importJSONElement(const std::string &name, const std: n["name"] << name; - bool isVariable = true; - bool isData = false; - // Check for the type of object, if it doesn't contain a type, it must be a variable - if (n.find("type")) { - isVariable = false; - std::string elementType = n.find("type")->val(); - if (elementType == "binned" || elementType == "unbinned") { - isData = true; - } - } - - // Import the object to the workspace - if (isVariable) { + // Without a type, it must be a variable. + JSONNode const *type = n.find("type"); + if (!type) { this->importVariableElement(n); - } else if (isData) { + } else if (type->val() == "binned" || type->val() == "unbinned") { HS3::Diagnostics diagnostics; - HS3::Data data; - readValue(HS3::Cursor{n, diagnostics}, data); + HS3::Data data = HS3::parseData(n, diagnostics); HS3::report(diagnostics); _workspace.import(*loadData(data, _workspace, nullptr), RooFit::Silence()); } else { @@ -4076,11 +4241,8 @@ void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) doc.domains.push_back({HS3::defaultDomainName, {std::move(axis)}}); doc.parameterPoints.push_back({"default_values", {parameter}}); + StateScope importScope{_import}; _import = std::make_unique(doc); - struct ImportScope { - RooJSONFactoryWSTool &tool; - ~ImportScope() { tool._import.reset(); } - } importScope{*this}; importVariable(parameter); if (RooRealVar *rrv = _workspace.var(parameter.name)) { @@ -4088,35 +4250,21 @@ void RooJSONFactoryWSTool::importVariableElement(const JSONNode &elementNode) } } -/** - * @brief Writes a warning message to the RooFit message service. - * - * @param str The warning message to be logged. - * @return std::ostream& A reference to the output stream. - */ +/// Write a warning to the RooFit message service, and return its stream. std::ostream &RooJSONFactoryWSTool::warning(std::string const &str) { return RooMsgService::instance().log(nullptr, RooFit::MsgLevel::WARNING, RooFit::ObjectHandling) << str << std::endl; } -/** - * @brief Writes an error message to the RooFit message service and throws a runtime_error. - * - * @param s The error message to be logged and thrown. - * @return void - */ +/// Write an error to the RooFit message service, and throw it as a std::runtime_error. void RooJSONFactoryWSTool::error(const char *s) { RooMsgService::instance().log(nullptr, RooFit::MsgLevel::ERROR, RooFit::IO) << s << std::endl; throw std::runtime_error(s); } -/** - * @brief Cleans up names to the HS3 standard - * - * @param str The string to be sanitized. - * @return std::string - */ +/// Replace some characters that are common in the names of RooFit objects, but that the HS3 standard doesn't allow in +/// names, if config().allowSanitizeNames is set. std::string RooJSONFactoryWSTool::sanitizeName(const std::string str) { std::string result; @@ -4146,8 +4294,6 @@ std::string RooJSONFactoryWSTool::sanitizeName(const std::string str) RooWorkspace RooJSONFactoryWSTool::cleanWS(const RooWorkspace &ws, bool onlyModelConfig) { - // Variables - RooWorkspace tmpWS = RooWorkspace(); if (onlyModelConfig) { for (auto *obj : ws.allGenericObjects()) { @@ -4202,36 +4348,22 @@ RooJSONFactoryWSTool::Config &RooJSONFactoryWSTool::config() // Sanitize all names in the workspace to be HS3 compliant RooWorkspace RooJSONFactoryWSTool::sanitizeWS(const RooWorkspace &ws) { - // Variables - RooWorkspace tmpWS = cleanWS(ws, false); - auto sanitizeIfNeeded = [](auto const &list) { - for (auto *obj : list) { - if (!isValidName(obj->GetName())) { - obj->SetName(sanitizeName(obj->GetName()).c_str()); - } - } - }; - sanitizeIfNeeded(tmpWS.allVars()); - sanitizeIfNeeded(tmpWS.allFunctions()); - sanitizeIfNeeded(tmpWS.allPdfs()); - sanitizeIfNeeded(tmpWS.allResolutionModels()); - // Datasets - for (auto *data : tmpWS.allData()) { - // Sanitize dataset name - if (!isValidName(data->GetName())) { - data->SetName(sanitizeName(data->GetName()).c_str()); - } - for (auto *obj : *data->get()) { + // Valid names stay as they are. + auto sanitize = [](auto const &objects) { + for (auto *obj : objects) { obj->SetName(sanitizeName(obj->GetName()).c_str()); } - } - for (auto *data : tmpWS.allEmbeddedData()) { - // Sanitize dataset name - data->SetName(sanitizeName(data->GetName()).c_str()); - for (auto *obj : *data->get()) { - obj->SetName(sanitizeName(obj->GetName()).c_str()); + }; + sanitize(tmpWS.allVars()); + sanitize(tmpWS.allFunctions()); + sanitize(tmpWS.allPdfs()); + sanitize(tmpWS.allResolutionModels()); + for (auto const &datasets : {tmpWS.allData(), tmpWS.allEmbeddedData()}) { + sanitize(datasets); + for (RooAbsData *data : datasets) { + sanitize(*data->get()); } } for (auto *snshObj : tmpWS.getSnapshots()) { @@ -4241,62 +4373,24 @@ RooWorkspace RooJSONFactoryWSTool::sanitizeWS(const RooWorkspace &ws) std::cerr << "Warning: found snapshot that is not a RooArgSet, skipping\n"; continue; } - - // Sanitize snapshot name - if (!isValidName(snsh->GetName())) { - snsh->setName(sanitizeName(snsh->GetName()).c_str()); - } - - // Sanitize the variables inside the snapshot - for (auto *arg : *snsh) { - if (!isValidName(arg->GetName())) { - arg->SetName(sanitizeName(arg->GetName()).c_str()); - } - } + snsh->setName(sanitizeName(snsh->GetName()).c_str()); + sanitize(*snsh); } - - // Generic objects (ModelConfigs, attributes, etc.) + // Generic objects, like the ModelConfigs with their sets for (auto *obj : tmpWS.allGenericObjects()) { - if (!isValidName(obj->GetName())) { - if (auto *named = dynamic_cast(obj)) { - named->SetName(sanitizeName(named->GetName()).c_str()); - } else { - std::cerr << "Warning: object " << obj->GetName() << " is not TNamed, cannot rename.\n"; - } + if (auto *named = dynamic_cast(obj)) { + named->SetName(sanitizeName(named->GetName()).c_str()); + } else if (!isValidName(obj->GetName())) { + std::cerr << "Warning: object " << obj->GetName() << " is not TNamed, cannot rename.\n"; } - if (auto *mc = dynamic_cast(obj)) { - // Sanitize ModelConfig name - if (!isValidName(mc->GetName())) { - mc->SetName(sanitizeName(mc->GetName()).c_str()); - } - - // Sanitize the sets inside ModelConfig - for (auto *obs : mc->GetObservables()->get()) { - if (obs) { - obs->SetName(sanitizeName(obs->GetName()).c_str()); - } - } - for (auto *poi : mc->GetParametersOfInterest()->get()) { - if (poi) { - poi->SetName(sanitizeName(poi->GetName()).c_str()); - } - } - for (auto *nuis : mc->GetNuisanceParameters()->get()) { - if (nuis) { - nuis->SetName(sanitizeName(nuis->GetName()).c_str()); - } - } - for (auto *glob : mc->GetGlobalObservables()->get()) { - if (glob) { - glob->SetName(sanitizeName(glob->GetName()).c_str()); - } - } + sanitize(*mc->GetObservables()); + sanitize(*mc->GetParametersOfInterest()); + sanitize(*mc->GetNuisanceParameters()); + sanitize(*mc->GetGlobalObservables()); } } - std::string wsName = std::string{ws.GetName()} + "_sanitized"; RooWorkspace newWS = cleanWS(tmpWS, false); - newWS.SetName(wsName.c_str()); - + newWS.SetName((std::string{ws.GetName()} + "_sanitized").c_str()); return newWS; } diff --git a/roofit/hs3/src/static_execute.h b/roofit/hs3/src/static_execute.h deleted file mode 100644 index ce980554ba5ac..0000000000000 --- a/roofit/hs3/src/static_execute.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Project: RooFit - * Authors: - * Jonas Rembser, CERN, Dec 2021 - * - * Copyright (c) 2022, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef static_execute_h -#define static_execute_h - -// Execute code on library loading by running code in the constructor of a -// class that is a static class member of another class. -#define STATIC_EXECUTE(MY_FUNC) \ - struct StaticExecutorWrapper { \ - struct Executor { \ - template \ - Executor(Func func) \ - { \ - func(); \ - } \ - }; \ - static Executor executor; \ - }; \ - \ - StaticExecutorWrapper::Executor StaticExecutorWrapper::executor{MY_FUNC}; - -#endif diff --git a/roofit/hs3/test/CMakeLists.txt b/roofit/hs3/test/CMakeLists.txt index 5edcc83457694..9e43a3ccd841a 100644 --- a/roofit/hs3/test/CMakeLists.txt +++ b/roofit/hs3/test/CMakeLists.txt @@ -1,4 +1,5 @@ ROOT_ADD_GTEST(testRooFitHS3 testRooFitHS3.cxx LIBRARIES RooFitCore RooFit RooFitHS3 RooStats HistFactory) +ROOT_ADD_GTEST(testHS3AuxiliaryMeasurements testHS3AuxiliaryMeasurements.cxx LIBRARIES RooFitCore RooFit RooFitHS3 RooStats HistFactory) if(test_roofit_hs3testsuite) include(FetchContent) diff --git a/roofit/hs3/test/HS3TestUtils.h b/roofit/hs3/test/HS3TestUtils.h new file mode 100644 index 0000000000000..cfa9b2d35e35d --- /dev/null +++ b/roofit/hs3/test/HS3TestUtils.h @@ -0,0 +1,1029 @@ +// Helpers for the tests of RooFitHS3 +// Authors: Carsten D. Burgard, DESY/ATLAS, 12/2021 +// Jonas Rembser, CERN 11/2026 + +#ifndef RooFitHS3_test_HS3TestUtils_h +#define RooFitHS3_test_HS3TestUtils_h + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace HS3Test { + +// --------------------------------------------------------------------------------------------------------------- +// Messages + +/// The tests print warnings and errors only, unless they change the level themselves. Register it with +/// `testing::AddGlobalTestEnvironment(new HS3Test::QuietEnvironment)`. +class QuietEnvironment : public testing::Environment { +public: + void SetUp() override { _level = std::make_unique(RooFit::WARNING); } + void TearDown() override { _level.reset(); } + +private: + std::unique_ptr _level; +}; + +/// Runs `action` and returns everything that was logged at the given message level and topic while it ran. +template +std::string captureMessages(RooFit::MsgLevel level, RooFit::MsgTopic topic, Action &&action) +{ + RooHelpers::HijackMessageStream stream{level, topic}; + action(); + return stream.str(); +} + +/// The warnings that the RooJSONFactoryWSTool gives while `action` runs. +template +std::string warningsOf(Action &&action) +{ + return captureMessages(RooFit::WARNING, RooFit::ObjectHandling, action); +} + +/// Like warningsOf(), but the warnings are logged again afterwards: they are still printed, and they still reach a +/// captureMessages() or warningsOf() around the call. +template +std::string recordWarningsOf(Action &&action) +{ + std::string warnings = warningsOf(action); + if (!warnings.empty()) { + RooMsgService::instance().log(nullptr, RooFit::WARNING, RooFit::ObjectHandling, /*forceSkipPrefix=*/true) + << warnings << std::flush; + } + return warnings; +} + +/// Counts the number of (non-overlapping) occurrences of `needle` in `haystack`. +inline std::size_t countOccurrences(std::string_view haystack, std::string_view needle) +{ + std::size_t result = 0; + for (std::size_t pos = 0; (pos = haystack.find(needle, pos)) != std::string_view::npos; pos += needle.size()) { + ++result; + } + return result; +} + +// --------------------------------------------------------------------------------------------------------------- +// Export and import + +/// A flag of RooJSONFactoryWSTool::config(), with the value that an export or import is done with. +using ConfigFlag = std::pair; + +// The flags by name, e.g. `toJSON(ws, {exportArrayDist(false), exportConstraintData(true)})`. +inline ConfigFlag exportArrayDist(bool value) +{ + return {&RooJSONFactoryWSTool::Config::exportArrayDist, value}; +} +inline ConfigFlag exportConstraintData(bool value) +{ + return {&RooJSONFactoryWSTool::Config::exportConstraintData, value}; +} +inline ConfigFlag renameCombinedObservables(bool value) +{ + return {&RooJSONFactoryWSTool::Config::renameCombinedObservables, value}; +} +inline ConfigFlag allowExportSharedVariates(bool value) +{ + return {&RooJSONFactoryWSTool::Config::allowExportSharedVariates, value}; +} +inline ConfigFlag identifyLegacyBinWidthFactors(bool value) +{ + return {&RooJSONFactoryWSTool::Config::identifyLegacyBinWidthFactors, value}; +} +inline ConfigFlag importConstraintsAsChannels(bool value) +{ + return {&RooJSONFactoryWSTool::Config::importConstraintsAsChannels, value}; +} + +/// Sets flags of RooJSONFactoryWSTool::config() for its lifetime. +class ScopedConfig { +public: + explicit ScopedConfig(std::vector const &flags) : _old{RooJSONFactoryWSTool::config()} + { + for (auto const &flag : flags) { + RooJSONFactoryWSTool::config().*flag.first = flag.second; + } + } + ~ScopedConfig() { RooJSONFactoryWSTool::config() = _old; } + ScopedConfig(ScopedConfig const &) = delete; + ScopedConfig &operator=(ScopedConfig const &) = delete; + +private: + RooJSONFactoryWSTool::Config _old; +}; + +/// The HS3 document that the workspace is exported to. +inline std::string toJSON(RooWorkspace &ws, std::vector const &flags = {}) +{ + ScopedConfig config{flags}; + return RooJSONFactoryWSTool{ws}.exportJSONtoString(); +} + +/// Imports an HS3 document into the workspace, and returns the warnings of the import. +inline std::string importInto(RooWorkspace &ws, std::string const &json, std::vector const &flags = {}) +{ + ScopedConfig config{flags}; + return recordWarningsOf([&] { RooJSONFactoryWSTool{ws}.importJSONfromString(json); }); +} + +/// The workspace that an HS3 document is imported to, with the warnings of the import. +struct Imported : RooWorkspace { + explicit Imported(std::string const &json, std::vector const &flags = {}) + : warnings{importInto(*this, json, flags)} + { + } + std::string warnings; +}; + +/// The document that RooFit writes for the workspace that it reads from `json`. +inline std::string reexport(std::string const &json, std::vector const &flags = {}) +{ + Imported ws{json}; + return toJSON(ws, flags); +} + +/// `text` with the first occurrence of `from`, which must exist, replaced by `to`. +inline std::string replaced(std::string text, std::string const &from, std::string const &to) +{ + const auto pos = text.find(from); + if (pos == std::string::npos) { + throw std::runtime_error("replaced(): no '" + from + "' in " + text); + } + return text.replace(pos, from.size(), to); +} + +/// The message of the exception that importing `json` throws, or an empty string if it is imported. +inline std::string importError(std::string const &json) +{ + RooWorkspace ws; + try { + importInto(ws, json); + } catch (std::exception const &e) { + return e.what(); + } + return ""; +} + +/// Asserts that importing `json` throws and logs an error message containing `expectedReason`. +inline void expectImportThrowsWithError(std::string const &json, std::string const &expectedReason) +{ + RooWorkspace ws; + bool threw = false; + const std::string errors = captureMessages(RooFit::ERROR, RooFit::IO, [&] { + try { + RooJSONFactoryWSTool{ws}.importJSONfromString(json); + } catch (const std::runtime_error &) { + threw = true; + } + }); + + EXPECT_TRUE(threw); + EXPECT_THAT(errors, testing::HasSubstr(expectedReason)); +} + +/// Asserts that exporting `ws` throws and logs an error message containing `expectedReason`. +inline void expectExportThrowsWithError(RooWorkspace &ws, std::string const &expectedReason) +{ + bool threw = false; + std::string exported; + const std::string errors = captureMessages(RooFit::ERROR, RooFit::IO, [&] { + try { + exported = toJSON(ws); + } catch (const std::runtime_error &) { + threw = true; + } + }); + + EXPECT_TRUE(threw); + EXPECT_TRUE(exported.empty()); + EXPECT_THAT(errors, testing::HasSubstr(expectedReason)); +} + +/// The object of that name and class in the workspace. Throws if there is none, which fails the test. +template +T *object(RooWorkspace &ws, std::string const &name) +{ + if (auto *out = dynamic_cast(ws.obj(name))) { + return out; + } + throw std::runtime_error("no " + std::string{T::Class_Name()} + " '" + name + "' in the workspace"); +} + +/// The ModelConfig of that name. Throws if there is none, which fails the test. +inline RooFit::ModelConfig &modelConfig(RooWorkspace &ws, std::string const &name) +{ + if (auto *mc = dynamic_cast(ws.obj(name))) { + return *mc; + } + throw std::runtime_error("no ModelConfig '" + name + "' in the workspace"); +} + +/// Imports a ModelConfig for the pdf into the workspace. The roles are comma-separated lists of names. +inline void addModelConfig(RooWorkspace &ws, const char *name, const char *pdf, const char *observables, + const char *poi, const char *nuisance = nullptr, const char *globals = nullptr) +{ + RooFit::ModelConfig mc{name, &ws}; + mc.SetPdf(pdf); + mc.SetObservables(observables); + mc.SetParametersOfInterest(poi); + if (nuisance) { + mc.SetNuisanceParameters(nuisance); + } + if (globals) { + mc.SetGlobalObservables(globals); + } + ws.import(mc); +} + +// --------------------------------------------------------------------------------------------------------------- +// Looking into documents + +/// An HS3 document with access to its nodes by path, like "distributions/model/samples/0/modifiers/lumi/data": the +/// steps are the keys of maps and, in lists, the position of an element or its "name". +class Doc { +public: + Doc(std::string json) : _json{std::move(json)} {} + Doc(const char *json) : _json{json} {} + /// The document that the workspace is exported to, with the warnings of the export. + explicit Doc(RooWorkspace &ws, std::vector const &flags = {}) + : _warnings{recordWarningsOf([&] { _json = toJSON(ws, flags); })} + { + } + + std::string const &json() const { return _json; } + operator std::string const &() const { return _json; } + std::string const &warnings() const { return _warnings; } + + RooFit::Detail::JSONNode const &root() const + { + if (!_tree) { + _tree = RooFit::Detail::JSONTree::create(_json); + } + return _tree->rootnode(); + } + + /// The node at the path, or null if there is none. + RooFit::Detail::JSONNode const *find(std::string const &path) const + { + RooFit::Detail::JSONNode const *node = &root(); + std::stringstream steps{path}; + for (std::string step; node && std::getline(steps, step, '/');) { + node = child(*node, step); + } + return node; + } + + /// The node at the path. Throws if there is none, which fails the test. + RooFit::Detail::JSONNode const &operator[](std::string const &path) const + { + if (auto const *node = find(path)) { + return *node; + } + throw std::runtime_error("no node '" + path + "' in " + _json); + } + + /// Whether the node at the path exists. The node that should contain it must exist. + bool has(std::string const &path) const + { + const auto pos = path.rfind('/'); + auto const &parent = pos == std::string::npos ? root() : (*this)[path.substr(0, pos)]; + return child(parent, path.substr(pos == std::string::npos ? 0 : pos + 1)) != nullptr; + } + + /// The values of the list at the path, with "null" for null. + std::vector strings(std::string const &path) const + { + std::vector out; + for (auto const &item : (*this)[path].children()) { + out.push_back(item.is_null() ? "null" : item.val()); + } + return out; + } + + /// The names of the objects in the list at the path. + std::vector names(std::string const &path) const + { + std::vector out; + for (auto const &item : (*this)[path].children()) { + out.push_back(item["name"].val()); + } + return out; + } + + /// The node at the path as JSON. + std::string dump(std::string const &path) const + { + std::stringstream ss; + (*this)[path].writeJSON(ss); + return ss.str(); + } + +private: + static RooFit::Detail::JSONNode const *child(RooFit::Detail::JSONNode const &node, std::string const &step) + { + if (node.is_map()) { + return node.find(step); + } + if (!node.is_seq()) { + return nullptr; + } + if (!step.empty() && step.find_first_not_of("0123456789") == std::string::npos) { + const std::size_t pos = std::stoul(step); + return pos < node.num_children() ? &node.child(pos) : nullptr; + } + return RooJSONFactoryWSTool::findNamedChild(node, step); + } + + std::string _json; + std::string _warnings; + mutable std::unique_ptr _tree; +}; + +namespace Detail { + +inline std::string dump(RooFit::Detail::JSONNode const &node) +{ + std::stringstream ss; + node.writeJSON(ss); + return ss.str(); +} + +// Describes in `why` the first difference between a node and the expectation, see matches(). +inline bool matchNode(RooFit::Detail::JSONNode const &actual, RooFit::Detail::JSONNode const &expected, + std::string const &path, std::string &why) +{ + auto mismatch = [&](std::string const &what) { + why = "at '" + path + "': expected " + what + ", found " + dump(actual); + return false; + }; + if (expected.is_map()) { + if (!actual.is_map()) { + return mismatch("a map"); + } + for (auto const &item : expected.children()) { + const std::string key = item.key(); + const bool absent = !item.is_container() && !item.is_null() && item.val() == ""; + if (absent ? actual.has_child(key) : !actual.has_child(key)) { + return mismatch(absent ? "no \"" + key + "\"" : "\"" + key + "\""); + } + if (!absent && !matchNode(actual[key], item, path + "/" + key, why)) { + return false; + } + } + return true; + } + if (expected.is_seq()) { + if (!actual.is_seq() || actual.num_children() != expected.num_children()) { + return mismatch("a list of " + std::to_string(expected.num_children()) + " elements"); + } + for (std::size_t i = 0; i < expected.num_children(); ++i) { + if (!matchNode(actual.child(i), expected.child(i), path + "/" + std::to_string(i), why)) { + return false; + } + } + return true; + } + bool equal = false; + if (expected.is_null() || actual.is_null() || actual.is_container()) { + equal = expected.is_null() && actual.is_null(); + } else if (expected.is_number() || actual.is_number()) { + equal = expected.is_number() && actual.is_number() && + testing::Matches(testing::DoubleEq(expected.val_double()))(actual.val_double()); + } else { + // Not `true` for "true": a boolean and a string are different values. + equal = dump(expected) == dump(actual); + } + return equal || mismatch(dump(expected)); +} + +} // namespace Detail + +/// Whether the node at the path matches the JSON value `expected`. Maps match if every key of `expected` matches: +/// other keys are ignored, and the value "" stands for a key that must not exist. Lists match element by +/// element, and other values if they are equal. +inline testing::AssertionResult matches(Doc const &doc, std::string const &path, std::string const &expected) +{ + auto const *node = doc.find(path); + if (!node) { + return testing::AssertionFailure() << "no node '" << path << "' in " << doc.json(); + } + auto expectedTree = RooFit::Detail::JSONTree::create(expected); + std::string why; + if (Detail::matchNode(*node, expectedTree->rootnode(), path, why)) { + return testing::AssertionSuccess(); + } + return testing::AssertionFailure() << why << "\nin '" << path << "': " << Detail::dump(*node); +} + +// --------------------------------------------------------------------------------------------------------------- +// Writing documents + +/// The non-empty parts, separated by commas. +inline std::string join(std::vector const &parts) +{ + std::string out; + for (auto const &part : parts) { + if (!part.empty()) { + out += (out.empty() ? "" : ", ") + part; + } + } + return out; +} + +/// An HS3 document of the given version with the given top-level sections. +inline std::string document(std::string const &version, std::vector const §ions = {}) +{ + return join({R"({"metadata": {"hs3_version": ")" + version + R"("})", join(sections)}) + "}"; +} + +/// A top-level section that is a list, like "distributions". +inline std::string section(std::string const &name, std::vector const &items) +{ + return "\"" + name + "\": [" + join(items) + "]"; +} + +inline std::string distributions(std::vector const &items) +{ + return section("distributions", items); +} + +inline std::string functions(std::vector const &items) +{ + return section("functions", items); +} + +/// The "domains" with the default domain of RooFit. +inline std::string defaultDomain(std::vector const &axes) +{ + return R"("domains": [{"name": "default_domain", "type": "product_domain", "axes": [)" + join(axes) + "]}]"; +} + +/// The "parameter_points" with the default values of RooFit. +inline std::string defaultValues(std::vector const ¶meters) +{ + return R"("parameter_points": [{"name": "default_values", "parameters": [)" + join(parameters) + "]}]"; +} + +namespace Detail { + +inline std::string number(double value) +{ + std::stringstream ss; + ss.precision(15); + ss << value; + const std::string out = ss.str(); + return out.find_first_of(".en") == std::string::npos ? out + ".0" : out; +} + +} // namespace Detail + +/// Axes with a common range for a domain. +inline std::string axes(std::vector const &names, double min, double max) +{ + std::vector out; + for (auto const &name : names) { + out.push_back(R"({"name": ")" + name + R"(", "min": )" + Detail::number(min) + R"(, "max": )" + + Detail::number(max) + "}"); + } + return join(out); +} + +/// The values of parameters for a parameter point. +inline std::string values(std::vector> const ¶meters) +{ + std::vector out; + for (auto const ¶meter : parameters) { + out.push_back(R"({"name": ")" + parameter.first + R"(", "value": )" + Detail::number(parameter.second) + "}"); + } + return join(out); +} + +/// A sample of a histfactory_dist, with the JSON of its data and the list of its modifiers. +inline std::string sample(std::string const &name, std::string const &data, std::string const &modifiers) +{ + return R"({"name": ")" + name + R"(", "data": )" + data + R"(, "modifiers": [)" + modifiers + "]}"; +} + +/// A histfactory_dist with the given samples and further keys, over the axis "x" (2 bins in [0, 2]) unless given. +inline std::string channel(std::string const &name, std::vector const &samples, + std::string const &keys = "", + std::string const &axes = R"({"name": "x", "min": 0.0, "max": 2.0, "nbins": 2})") +{ + return join({R"({"name": ")" + name + R"(", "type": "histfactory_dist", "axes": [)" + axes + "]", keys, + R"("samples": [)" + join(samples) + "]}"}); +} + +/// A document with the histfactory_dist "model_channel0" on the axis "x" (2 bins in [0, 2]), which has one sample +/// "sig" with the contents [10, 20] and the given modifiers. +class HistFactoryDoc { +public: + HistFactoryDoc(std::string version, std::string modifiers) + : _version{std::move(version)}, _modifiers{std::move(modifiers)} + { + } + /// The "default_interpolation" of the channel. + HistFactoryDoc &interpolation(std::string const &value) + { + return channelKeys("\"default_interpolation\": " + value); + } + /// The "data" of the sample. + HistFactoryDoc &sampleData(std::string value) + { + _data = std::move(value); + return *this; + } + /// Further keys of the channel. + HistFactoryDoc &channelKeys(std::string const &value) + { + _keys.push_back(value); + return *this; + } + /// Further distributions. + HistFactoryDoc &distributions(std::string const &value) + { + _distributions.push_back(value); + return *this; + } + /// Further top-level sections, like defaultDomain(). + HistFactoryDoc §ions(std::string const &value) + { + _sections.push_back(value); + return *this; + } + std::string str() const + { + std::vector dists{channel("model_channel0", {sample("sig", _data, _modifiers)}, join(_keys))}; + dists.insert(dists.end(), _distributions.begin(), _distributions.end()); + std::vector all{HS3Test::distributions(dists)}; + all.insert(all.end(), _sections.begin(), _sections.end()); + return document(_version, all); + } + operator std::string() const { return str(); } + +private: + std::string _version; + std::string _modifiers; + std::string _data = R"({"contents": [10.0, 20.0]})"; + std::vector _keys; + std::vector _distributions; + std::vector _sections; +}; + +// --------------------------------------------------------------------------------------------------------------- +// Round trips + +/// Flags that a round trip is done with, see constraintStyles(). +struct RoundTripStyle { + std::string name; + std::vector flags; +}; + +/// A model whose round trip through HS3 is tested, see expectRoundTrip() and the suite RoundTrip of +/// testRooFitHS3.cxx. +struct RoundTripModel { + using Builder = std::function; + + /// The options for expectRoundTrip() of a workspace that is built elsewhere. + RoundTripModel() : name{"workspace"} {} + /// A model that the function builds in an empty workspace. + RoundTripModel(std::string name_, Builder build_) : name{std::move(name_)}, build{std::move(build_)} {} + /// A model made with the workspace factory. + RoundTripModel(std::string name_, std::vector const &factoryExpressions) + : name{std::move(name_)}, build{[=](RooWorkspace &ws) { + for (auto const &expression : factoryExpressions) { + if (!ws.factory(expression)) { + throw std::runtime_error("factory expression failed: " + expression); + } + } + }} + { + } + /// A model that is imported from an HS3 document. + static RoundTripModel fromDocument(std::string name, std::string const &json) + { + return {std::move(name), [=](RooWorkspace &ws) { importInto(ws, json); }}; + } + + /// The flags that the round trip is done with. By default, only without flags. + RoundTripModel with(std::vector value) const + { + RoundTripModel out{*this}; + out.styles = std::move(value); + return out; + } + /// The observables: the values of the functions are compared across their ranges, see sameValues(). + RoundTripModel over(std::vector value) const + { + RoundTripModel out{*this}; + out.observables = std::move(value); + return out; + } + /// The ModelConfig and the data set whose likelihood is compared, see likelihoodScan(). + RoundTripModel likelihoodOf(std::string modelConfig_, std::string data_) const + { + RoundTripModel out{*this}; + out.modelConfig = std::move(modelConfig_); + out.data = std::move(data_); + return out; + } + /// A function that gives numbers for a workspace, which are compared as well: e.g. a likelihood at chosen + /// points, or yields. + RoundTripModel comparing(std::function(RooWorkspace &)> value) const + { + RoundTripModel out{*this}; + out.numbers = std::move(value); + return out; + } + /// The values agree up to this relative tolerance, instead of up to 4 ULPs as for EXPECT_DOUBLE_EQ. + RoundTripModel within(double value) const + { + RoundTripModel out{*this}; + out.tolerance = value; + return out; + } + /// The second workspace doesn't have the same components as the first one, e.g. because the import names them + /// differently. The functions that it has are still compared. + RoundTripModel withOtherComponents() const + { + RoundTripModel out{*this}; + out.sameComponents = false; + return out; + } + /// The first document differs from the following ones, e.g. because the import adds attributes. + RoundTripModel stableFromSecondExport() const + { + RoundTripModel out{*this}; + out.firstExportDiffers = true; + return out; + } + /// Evaluating the model changes what is exported, e.g. because it leaves a named range on a variable. + RoundTripModel exportChangesAfterEvaluation() const + { + RoundTripModel out{*this}; + out.stableAfterEvaluation = false; + return out; + } + + std::string name; + Builder build; + std::vector styles{{"Default", {}}}; + std::vector observables{"x"}; + std::string modelConfig; + std::string data; + std::function(RooWorkspace &)> numbers; + double tolerance = 0.; + bool sameComponents = true; + bool firstExportDiffers = false; + bool stableAfterEvaluation = true; +}; + +inline void PrintTo(RoundTripModel const &model, std::ostream *os) +{ + *os << model.name; +} + +/// The four layouts of the constraint terms that the export flags select: as array distributions or on their own, +/// in "aux_distributions" or paired with data sets. +inline std::vector constraintStyles() +{ + return {{"ArrayAux", {}}, + {"ArrayData", {exportConstraintData(true)}}, + {"PlainAux", {exportArrayDist(false)}}, + {"PlainData", {exportArrayDist(false), exportConstraintData(true)}}}; +} + +namespace Detail { + +inline bool sameValue(double a, double b, double tolerance) +{ + if (std::isnan(a) || std::isnan(b)) { + return std::isnan(a) && std::isnan(b); + } + return tolerance == 0. ? testing::Matches(testing::DoubleEq(a))(b) + : std::abs(a - b) <= tolerance * std::max(std::abs(a), std::abs(b)); +} + +// The floating variables of the workspace that are no observables, with values that are moved from the current +// ones by a fraction, in a direction that depends on the name. +inline std::map +displacedValues(RooWorkspace &ws, std::vector const &observables, double fraction) +{ + std::map out; + for (auto *var : static_range_cast(ws.allVars())) { + const std::string name = var->GetName(); + if (var->isConstant() || std::find(observables.begin(), observables.end(), name) != observables.end()) { + continue; + } + // A fraction of the value, or of the range if the value is zero. + double scale = var->getVal() != 0. ? std::abs(var->getVal()) : 1.; + if (var->hasMin() && var->hasMax()) { + scale = var->getVal() != 0. ? std::min(scale, var->getMax() - var->getMin()) : var->getMax() - var->getMin(); + } + const int sign = (name.size() + name.back()) % 2 ? 1 : -1; + double value = var->getVal() + sign * fraction * scale; + if (!var->inRange(value, nullptr)) { + value = var->getVal() - sign * fraction * scale; + } + if (var->inRange(value, nullptr)) { + out[name] = value; + } + } + return out; +} + +inline void setValues(RooWorkspace &ws, std::map const &values) +{ + for (auto const &item : values) { + if (RooRealVar *var = ws.var(item.first)) { + var->setVal(item.second); + } + } +} + +} // namespace Detail + +/// Whether the functions and pdfs that both workspaces have give the same values: in every bin of each observable +/// (at ten points across its range, if it has no bins), for every state of each category, with the parameters as +/// they are and with the floating parameters moved. The pdfs are normalized over the observables. With +/// `sameComponents`, the workspaces must have the same components. +inline testing::AssertionResult +sameValues(RooWorkspace &ws1, RooWorkspace &ws2, RoundTripModel const &model, bool sameComponents) +{ + if (sameComponents) { + std::vector names1; + std::vector names2; + for (RooAbsArg *arg : ws1.components()) { + names1.push_back(arg->GetName()); + } + for (RooAbsArg *arg : ws2.components()) { + names2.push_back(arg->GetName()); + } + std::sort(names1.begin(), names1.end()); + std::sort(names2.begin(), names2.end()); + if (names1 != names2) { + return testing::AssertionFailure() << "the components differ: " << testing::PrintToString(names1) << " and " + << testing::PrintToString(names2); + } + } + + // The values of the observables and the states of the categories are not part of a document: start from the ones + // of the first workspace. + RooArgSet nset1; + RooArgSet nset2; + std::vector> observables; + for (auto const &name : model.observables) { + if (ws1.var(name) && ws2.var(name)) { + observables.emplace_back(ws1.var(name), ws2.var(name)); + nset1.add(*ws1.var(name)); + nset2.add(*ws2.var(name)); + ws2.var(name)->setVal(ws1.var(name)->getVal()); + } + } + for (auto *category : static_range_cast(ws1.allCats())) { + RooCategory *other = ws2.cat(category->GetName()); + if (other && other->hasLabel(category->getCurrentLabel())) { + other->setLabel(category->getCurrentLabel()); + } + } + std::vector> functions; + RooArgSet all{ws1.allPdfs()}; + all.add(ws1.allFunctions()); + for (auto *function : static_range_cast(all)) { + if (auto *other = dynamic_cast(ws2.arg(function->GetName()))) { + functions.emplace_back(function, other); + } + } + if (!all.empty() && functions.empty()) { + return testing::AssertionFailure() << "the second workspace has none of the functions of the first one"; + } + std::size_t nCompared = 0; + std::string where; + auto compare = [&]() -> testing::AssertionResult { + for (auto const &pair : functions) { + const double value1 = pair.first->getVal(nset1); + const double value2 = pair.second->getVal(nset2); + ++nCompared; + if (!Detail::sameValue(value1, value2, model.tolerance)) { + return testing::AssertionFailure() << "'" << pair.first->GetName() << "' is " << value1 << " and " << value2 + << " (difference " << value2 - value1 << ") " << where; + } + } + return testing::AssertionSuccess(); + }; + auto scan = [&]() -> testing::AssertionResult { + where = ""; + if (auto result = compare(); !result) { + return result; + } + for (auto const &observable : observables) { + RooRealVar &var = *observable.first; + const double old = var.getVal(); + // The centers of the bins. A variable has no bins unless they were set: then points across its range. + std::vector points; + for (int i = 0; i < var.numBins(); ++i) { + points.push_back(var.getBinning().binCenter(i)); + } + if (points.empty() && var.hasMin() && var.hasMax()) { + const int n = 10; + for (int i = 0; i < n; ++i) { + points.push_back(var.getMin() + (i + 0.5) / n * (var.getMax() - var.getMin())); + } + } + for (double point : points) { + var.setVal(point); + observable.second->setVal(point); + where = "at " + std::string(var.GetName()) + " = " + std::to_string(point); + if (auto result = compare(); !result) { + return result; + } + } + var.setVal(old); + observable.second->setVal(old); + } + for (auto *category : static_range_cast(ws1.allCats())) { + RooCategory *other = ws2.cat(category->GetName()); + if (!other) { + continue; + } + const std::string old = category->getCurrentLabel(); + for (auto const &state : *category) { + if (!other->hasLabel(state.first)) { + return testing::AssertionFailure() + << "'" << category->GetName() << "' has no state '" << state.first << "'"; + } + category->setLabel(state.first.c_str()); + other->setLabel(state.first.c_str()); + where = "for the state '" + state.first + "' of '" + category->GetName() + "'"; + if (auto result = compare(); !result) { + return result; + } + } + category->setLabel(old.c_str()); + other->setLabel(old.c_str()); + } + return testing::AssertionSuccess(); + }; + + if (auto result = scan(); !result) { + return result; + } + const auto displaced = Detail::displacedValues(ws1, model.observables, 0.05); + std::map original; + for (auto const &item : displaced) { + original[item.first] = ws1.var(item.first)->getVal(); + } + Detail::setValues(ws1, displaced); + Detail::setValues(ws2, displaced); + auto result = scan(); + Detail::setValues(ws1, original); + Detail::setValues(ws2, original); + if (!result) { + return result << " with moved parameters"; + } + if (!functions.empty() && nCompared == 0) { + return testing::AssertionFailure() << "no values compared"; + } + return testing::AssertionSuccess(); +} + +/// The NLL of a ModelConfig for a data set, with its external constraints and global observables: at the values +/// of the parameters in the workspace, and with the floating parameters moved in both directions. +inline std::vector likelihoodScan(RooWorkspace &ws, std::string const &mcName, std::string const &data) +{ + RooFit::ModelConfig *mc = &modelConfig(ws, mcName); + if (!mc->GetPdf() || !ws.data(data)) { + throw std::runtime_error("likelihoodScan(): '" + mcName + "' has no pdf, or there is no data '" + data + "'"); + } + RooArgSet external; + if (mc->GetExternalConstraints()) { + external.add(*mc->GetExternalConstraints()); + } + RooArgSet globals; + if (mc->GetGlobalObservables()) { + globals.add(*mc->GetGlobalObservables()); + } + // An empty set of global observables is not the same as none. + const RooCmdArg externalArg = external.empty() ? RooCmdArg::none() : RooFit::ExternalConstraints(external); + const RooCmdArg globalsArg = globals.empty() ? RooCmdArg::none() : RooFit::GlobalObservables(globals); + std::unique_ptr nll{mc->GetPdf()->createNLL(*ws.data(data), externalArg, globalsArg)}; + std::vector observables; + for (RooAbsArg *arg : *ws.data(data)->get()) { + observables.push_back(arg->GetName()); + } + for (RooAbsArg *arg : globals) { + observables.push_back(arg->GetName()); + } + std::map original; + for (auto const &item : Detail::displacedValues(ws, observables, 0.)) { + original[item.first] = item.second; + } + std::vector out{nll->getVal()}; + for (double fraction : {0.03, -0.06}) { + Detail::setValues(ws, original); + Detail::setValues(ws, Detail::displacedValues(ws, observables, fraction)); + out.push_back(nll->getVal()); + } + Detail::setValues(ws, original); + return out; +} + +/// Whether two lists of values, like the ones of likelihoodScan(), agree: up to 4 ULPs, or up to a relative +/// tolerance if one is given. +inline testing::AssertionResult +sameNumbers(std::vector const &a, std::vector const &b, double tolerance) +{ + if (a.size() != b.size() || a.empty()) { + return testing::AssertionFailure() << a.size() << " and " << b.size() << " values"; + } + for (std::size_t i = 0; i < a.size(); ++i) { + if (!std::isfinite(a[i]) || !Detail::sameValue(a[i], b[i], tolerance)) { + return testing::AssertionFailure() << "value " << i << " is " << testing::PrintToString(a[i]) << " and " + << testing::PrintToString(b[i]) << " (difference " << b[i] - a[i] << ")"; + } + } + return testing::AssertionSuccess(); +} + +/// The round trip of a workspace for each style of the model: the workspace that is read from the exported +/// document is exported to the same document again, also after it was evaluated; it has the same components +/// (without flags); and its functions and likelihood have the same values, see sameValues(), likelihoodScan() and +/// RoundTripModel::comparing(). +inline void expectRoundTrip(RooWorkspace &ws, RoundTripModel const &model) +{ + for (auto const &style : model.styles) { + SCOPED_TRACE(model.name + ", " + style.name); + const std::string json = toJSON(ws, style.flags); + Imported ws2{json}; + const std::string json2 = toJSON(ws2, style.flags); + if (!model.firstExportDiffers) { + EXPECT_EQ(json2, json); + } + EXPECT_EQ(reexport(json2, style.flags), json2); + EXPECT_TRUE(sameValues(ws, ws2, model, model.sameComponents && style.flags.empty())); + if (model.numbers) { + // The function may move the variables. + RooArgSet variables{ws.allVars()}; + variables.add(ws.allCats()); + RooArgSet variables2{ws2.allVars()}; + variables2.add(ws2.allCats()); + std::unique_ptr values{variables.snapshot()}; + std::unique_ptr values2{variables2.snapshot()}; + EXPECT_TRUE(sameNumbers(model.numbers(ws), model.numbers(ws2), model.tolerance)); + variables.assign(*values); + variables2.assign(*values2); + } + if (!model.data.empty()) { + EXPECT_TRUE(sameNumbers(likelihoodScan(ws, model.modelConfig, model.data), + likelihoodScan(ws2, model.modelConfig, model.data), model.tolerance)); + } + // The evaluations leave objects like normalization integrals behind, which must not change the export. + if (model.stableAfterEvaluation) { + EXPECT_EQ(toJSON(ws2, style.flags), json2); + } + } +} + +/// The round trip of a workspace without flags. +inline void expectRoundTrip(RooWorkspace &ws) +{ + expectRoundTrip(ws, RoundTripModel{}); +} + +/// The round trip of a workspace with only this function or pdf. +inline void expectRoundTrip(RooAbsArg const &arg) +{ + RooWorkspace ws; + ws.import(arg, RooFit::Silence()); + expectRoundTrip(ws, RoundTripModel{arg.GetName(), RoundTripModel::Builder{}}); +} + +} // namespace HS3Test + +#endif diff --git a/roofit/hs3/test/testHS3AuxiliaryMeasurements.cxx b/roofit/hs3/test/testHS3AuxiliaryMeasurements.cxx new file mode 100644 index 0000000000000..f0d21012987ed --- /dev/null +++ b/roofit/hs3/test/testHS3AuxiliaryMeasurements.cxx @@ -0,0 +1,1402 @@ +// Tests for the auxiliary measurements (constraint terms) of HistFactory models in HS3 +// Authors: Carsten D. Burgard, DESY/CMS, 11/2026 + +// A classical HistFactory workspace is written to HS3 in every layout of the auxiliary measurements that the flags +// RooJSONFactoryWSTool::Config::exportArrayDist and ::exportConstraintData select, and read back. The model must stay +// the same: the same likelihood, the same toys, and the same document when it is written again, also if the layout +// changes between the exports. + +#include "HS3TestUtils.h" + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using RooFit::Detail::JSONNode; +using RooFit::Detail::JSONTree; + +namespace { + +// The four combinations of the two export flags. The HS3 documentation shows three layouts for auxiliary +// measurements: the terms in "aux_distributions" (chapter Likelihoods), each constraint paired with its own data set, +// and one array distribution paired with one unbinned data set (both in chapter Auxiliary measurements). +struct Style { + const char *name; + bool arrays; // config().exportArrayDist + bool constraintData; // config().exportConstraintData +}; + +const std::vector