From b5e19a6e303d9f6ef3a5b2f19c9947342eb3f306 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Thu, 19 Feb 2026 16:07:56 +0100 Subject: [PATCH 1/6] [RF] Optimizations to avoid taking logarithms of Gaussian constraints Large likelihood fits (e.g. in CMS and ATLAS) often contain many Gaussian constraint terms. In the standard evaluation, these incur unnecessary overhead by computing the PDF value via `exp(...)` only to immediately take the logarithm in the constraint term. This commit introduces a log-space optimization that avoids the redundant exp -> log roundtrip. A class that can output its logarithm overrides the new RooAbsArg::canOutputLogarithm(). The Evaluator decides for which nodes this is used, and doEval() reads the decision from the new EvalContext::outputsLogarithm(). Evaluator::print() shows it in a new column, because the value printed for such a node is a logarithm. We need this optimization because it makes a difference for CMS, and they have also implemented something like this in the custom CMS Combine likelihood classes. If we want them to gradually move to "vanilla" RooFit, we need at least the same optimizations. Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit 24ac3218a583409e64d05a1fcb38245438fe298a) --- roofit/batchcompute/res/RooBatchCompute.h | 1 + roofit/batchcompute/src/ComputeFunctions.cxx | 13 +++++ roofit/roofit/inc/RooGaussian.h | 1 + roofit/roofit/src/RooGaussian.cxx | 20 +++++-- roofit/roofitcore/inc/RooAbsArg.h | 7 +++ .../roofitcore/inc/RooFit/Detail/MathFuncs.h | 12 +++-- .../inc/RooFit/Detail/RooNLLVarNew.h | 7 ++- .../inc/RooFit/Detail/RooNormalizedPdf.h | 1 + roofit/roofitcore/inc/RooFit/EvalContext.h | 8 +++ roofit/roofitcore/inc/RooRealIntegral.h | 3 ++ roofit/roofitcore/src/RooConstraintSum.cxx | 7 ++- roofit/roofitcore/src/RooFit/EvalContext.cxx | 1 + roofit/roofitcore/src/RooFit/Evaluator.cxx | 54 +++++++++++++++++-- roofit/roofitcore/src/RooNLLVarNew.cxx | 3 +- roofit/roofitcore/src/RooNormalizedPdf.cxx | 10 ++++ roofit/roofitcore/src/RooRealIntegral.cxx | 12 +++++ 16 files changed, 146 insertions(+), 14 deletions(-) diff --git a/roofit/batchcompute/res/RooBatchCompute.h b/roofit/batchcompute/res/RooBatchCompute.h index 004207b22ff37..8297967e98980 100644 --- a/roofit/batchcompute/res/RooBatchCompute.h +++ b/roofit/batchcompute/res/RooBatchCompute.h @@ -97,6 +97,7 @@ enum Computer { Gamma, GaussModelExpBasis, Gaussian, + LogGaussian, Identity, Johnson, Landau, diff --git a/roofit/batchcompute/src/ComputeFunctions.cxx b/roofit/batchcompute/src/ComputeFunctions.cxx index e1ae37eb1137a..febc7e71b0911 100644 --- a/roofit/batchcompute/src/ComputeFunctions.cxx +++ b/roofit/batchcompute/src/ComputeFunctions.cxx @@ -465,6 +465,18 @@ __rooglobal__ void computeGaussian(Batches &batches) } } +__rooglobal__ void computeLogGaussian(Batches &batches) +{ + auto x = batches.args[0]; + auto mean = batches.args[1]; + auto sigma = batches.args[2]; + for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { + const double arg = x[i] - mean[i]; + const double halfBySigmaSq = -0.5 / (sigma[i] * sigma[i]); + batches.output[i] = arg * arg * halfBySigmaSq; + } +} + __rooglobal__ void computeIdentity(Batches &batches) { for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { @@ -944,6 +956,7 @@ std::vector getFunctions() computeGamma, computeGaussModelExpBasis, computeGaussian, + computeLogGaussian, computeIdentity, computeJohnson, computeLandau, diff --git a/roofit/roofit/inc/RooGaussian.h b/roofit/roofit/inc/RooGaussian.h index 465b992cb70bc..ee024dd0ffb6b 100644 --- a/roofit/roofit/inc/RooGaussian.h +++ b/roofit/roofit/inc/RooGaussian.h @@ -58,6 +58,7 @@ class RooGaussian : public RooAbsPdf { double evaluate() const override; void doEval(RooFit::EvalContext &) const override; + bool canOutputLogarithm() const override { return true; } inline bool canComputeBatchWithCuda() const override { return true; } private: diff --git a/roofit/roofit/src/RooGaussian.cxx b/roofit/roofit/src/RooGaussian.cxx index f9741f66c3ffb..f991b34fe90e1 100644 --- a/roofit/roofit/src/RooGaussian.cxx +++ b/roofit/roofit/src/RooGaussian.cxx @@ -27,8 +27,6 @@ Plain Gaussian p.d.f #include -#include - //////////////////////////////////////////////////////////////////////////////// @@ -61,10 +59,22 @@ double RooGaussian::evaluate() const //////////////////////////////////////////////////////////////////////////////// /// Compute multiple values of Gaussian distribution. -void RooGaussian::doEval(RooFit::EvalContext & ctx) const +void RooGaussian::doEval(RooFit::EvalContext &ctx) const { - RooBatchCompute::compute(ctx.config(this), RooBatchCompute::Gaussian, ctx.output(), - {ctx.at(x), ctx.at(mean), ctx.at(sigma)}); + if (ctx.outputsLogarithm(this)) { + auto output = ctx.output(); + if (output.size() == 1) { + // If the output size is just one, which is common for constraints, + // calling into RooBatchCompute is not worth its overhead. + output[0] = RooFit::Detail::MathFuncs::logGaussian(ctx.at(x)[0], ctx.at(mean)[0], ctx.at(sigma)[0]); + } else { + RooBatchCompute::compute(ctx.config(this), RooBatchCompute::LogGaussian, output, + {ctx.at(x), ctx.at(mean), ctx.at(sigma)}); + } + return; + } + RooBatchCompute::compute(ctx.config(this), RooBatchCompute::Gaussian, ctx.output(), + {ctx.at(x), ctx.at(mean), ctx.at(sigma)}); } //////////////////////////////////////////////////////////////////////////////// diff --git a/roofit/roofitcore/inc/RooAbsArg.h b/roofit/roofitcore/inc/RooAbsArg.h index bdc29c720b296..b0237579be50f 100644 --- a/roofit/roofitcore/inc/RooAbsArg.h +++ b/roofit/roofitcore/inc/RooAbsArg.h @@ -491,6 +491,13 @@ class RooAbsArg : public TNamed, public RooPrintable { return false; }; + // Information to expose to the RooFit::Evaluator for optimized evaluation: + + /// Whether doEval() can output the logarithm of the value instead of the + /// value. It then has to do so if `ctx.outputsLogarithm(this)` is true. A + /// derived class that overrides doEval() without supporting this must also + /// override this function to return false. + virtual bool canOutputLogarithm() const { return false; } virtual bool canComputeBatchWithCuda() const { return false; } virtual bool isReducerNode() const { return false; } diff --git a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h index 2e6f8deef931c..7afba26486d4f 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h +++ b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h @@ -84,12 +84,18 @@ double bernstein(double x, double xmin, double xmax, DoubleArray coefs, int nCoe return result; } -/// @brief Function to evaluate an un-normalized RooGaussian. -inline double gaussian(double x, double mean, double sigma) +/// Evaluate the logarithm of an un-normalized Gaussian. +inline double logGaussian(double x, double mean, double sigma) { const double arg = x - mean; const double sig = sigma; - return std::exp(-0.5 * arg * arg / (sig * sig)); + return -0.5 * arg * arg / (sig * sig); +} + +/// @brief Function to evaluate an un-normalized Gaussian. +inline double gaussian(double x, double mean, double sigma) +{ + return std::exp(logGaussian(x, mean, sigma)); } template diff --git a/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h b/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h index 98c31a66d1e96..6e0f17fb3b4fa 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h +++ b/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h @@ -71,7 +71,11 @@ class RooNLLVarNew : public RooAbsReal { void enableBinOffsetting(bool on = true) { _doBinOffset = on; } - void setSimCount(int simCount) { _simCount = simCount; } + void setSimCount(int simCount) + { + _simCount = simCount; + _logSimCount = std::log(static_cast(simCount)); + } enum class FuncMode { Pdf, @@ -118,6 +122,7 @@ class RooNLLVarNew : public RooAbsReal { FuncMode _funcMode = FuncMode::Pdf; RooDataHist::ErrorType _chi2ErrorType = RooDataHist::Expected; int _simCount = 1; + double _logSimCount = 0.; std::string _prefix; std::vector _binw; mutable ROOT::Math::KahanSum _offset{0.}; ///createExpectedEventsFunc(&_normSet); } + bool canOutputLogarithm() const override { return true; } bool canComputeBatchWithCuda() const override { return true; } RooAbsPdf const &pdf() const { return *_pdf; } diff --git a/roofit/roofitcore/inc/RooFit/EvalContext.h b/roofit/roofitcore/inc/RooFit/EvalContext.h index 5940dd281a981..eea04eb16466a 100644 --- a/roofit/roofitcore/inc/RooFit/EvalContext.h +++ b/roofit/roofitcore/inc/RooFit/EvalContext.h @@ -109,6 +109,13 @@ class EvalContext { RooBatchCompute::Config config(RooAbsArg const *arg) const; + /// Whether the output of the given node is the logarithm of its value. See + /// RooAbsArg::canOutputLogarithm(). + bool outputsLogarithm(RooAbsArg const *arg) const + { + return arg->hasDataToken() && _outputsLogarithm[arg->dataToken()]; + } + /// A counter that is incremented every time new input data is loaded into /// the evaluation context. Reducer nodes can use it as a cache /// invalidation key for quantities that only depend on the input data, @@ -142,6 +149,7 @@ class EvalContext { std::vector> _buffers; std::size_t _bufferIdx = 0; std::vector _cfgs; + std::vector _outputsLogarithm; std::vector> _deferredActions; }; diff --git a/roofit/roofitcore/inc/RooRealIntegral.h b/roofit/roofitcore/inc/RooRealIntegral.h index 04e1b9d9c8573..d84df3b73053b 100644 --- a/roofit/roofitcore/inc/RooRealIntegral.h +++ b/roofit/roofitcore/inc/RooRealIntegral.h @@ -103,6 +103,9 @@ class RooRealIntegral : public RooAbsReal { double evaluate() const override ; bool isValidReal(double value, bool printError=false) const override ; + void doEval(RooFit::EvalContext &) const override; + bool canOutputLogarithm() const override { return true; } + bool redirectServersHook(const RooAbsCollection& newServerList, bool mustReplaceAll, bool nameChange, bool isRecursive) override ; diff --git a/roofit/roofitcore/src/RooConstraintSum.cxx b/roofit/roofitcore/src/RooConstraintSum.cxx index 27a6cc4acea9e..b40b767640659 100644 --- a/roofit/roofitcore/src/RooConstraintSum.cxx +++ b/roofit/roofitcore/src/RooConstraintSum.cxx @@ -78,12 +78,17 @@ double RooConstraintSum::evaluate() const return sum; } +/// Evaluate with the vectorizing CPU backend. void RooConstraintSum::doEval(RooFit::EvalContext &ctx) const { double sum(0); for (const auto comp : _set1) { - sum -= std::log(ctx.at(comp)[0]); + // We only need to take the logarithm if the server didn't do it already: + if (!ctx.outputsLogarithm(comp)) + sum -= std::log(ctx.at(comp)[0]); + else + sum -= ctx.at(comp)[0]; } ctx.output()[0] = sum; diff --git a/roofit/roofitcore/src/RooFit/EvalContext.cxx b/roofit/roofitcore/src/RooFit/EvalContext.cxx index 9d931ae2146c0..aad0c7ed80a5c 100644 --- a/roofit/roofitcore/src/RooFit/EvalContext.cxx +++ b/roofit/roofitcore/src/RooFit/EvalContext.cxx @@ -83,6 +83,7 @@ void EvalContext::resize(std::size_t n) { _cfgs.resize(n); _ctx.resize(n); + _outputsLogarithm.resize(n); } /// \brief Sets the output value with an offset. diff --git a/roofit/roofitcore/src/RooFit/Evaluator.cxx b/roofit/roofitcore/src/RooFit/Evaluator.cxx index 6a807498e76bf..c098950af5132 100644 --- a/roofit/roofitcore/src/RooFit/Evaluator.cxx +++ b/roofit/roofitcore/src/RooFit/Evaluator.cxx @@ -33,6 +33,8 @@ RooAbsPdf::fitTo() is called and gets destroyed when the fitting ends. #include #include #include +#include +#include #include #include #include @@ -221,6 +223,50 @@ Evaluator::Evaluator(const RooAbsReal &absReal, bool useGPU) syncDataTokens(); + // This is an optimization for Gaussian constraint terms, of which there are + // many in big fits. To Avoid taking the the exponential and then the + // logarithm for the constraint term, we request that all normalized pdfs + // that have only the RooConstraintSum as a value client are evaluating + // their logarithms, which the RooConstrantSum::doEval() then assumes. + // Note that we don't need to consider the GPU evaluation path here, because + // scalar terms like the constraints are always evaluated on the CPU. + for (NodeInfo &info : _nodes) { + if (!dynamic_cast(info.absArg)) { + continue; + } + for (RooAbsArg *server : info.absArg->servers()) { + if (!server->canOutputLogarithm()) { + continue; + } + // If the pdf has clients other than the RooConstraintSum, we can't do + // the log optimization. + if (server->clients().size() != 1) { + continue; + } + auto &outputsLogarithm = _evalContextCPU._outputsLogarithm; + if (auto normPdf = dynamic_cast(server)) { + // If this is a normalized pdf, we also request the log of the pdf + // value and normalization integral. + // Disable the optimization if the computation graph doesn't have + // the expected structure, for safety. The unnormalized pdf has two + // clients: the integral, and the RooNormalizedPdf. + bool const canOptimizePdf = normPdf->pdf().canOutputLogarithm() && normPdf->pdf().clients().size() == 2; + bool const canOptimizeNormIntegral = + normPdf->normIntegral().canOutputLogarithm() && normPdf->normIntegral().clients().size() == 1; + // If the servers can't be optimized, no gain in optimizing the + // normalized pdf. + if (!canOptimizePdf || !canOptimizeNormIntegral) { + continue; + } + // Only now set the flags, such that the log optimization for + // the normalized pdf and its servers is enabled atomically. + outputsLogarithm[normPdf->pdf().dataToken()] = true; + outputsLogarithm[normPdf->normIntegral().dataToken()] = true; + } + outputsLogarithm[server->dataToken()] = true; + } + } + if (_useGPU) { // Create the single CUDA stream on which all GPU computations and data // transfers of this Evaluator are enqueued. The graph is evaluated in @@ -728,7 +774,7 @@ void Evaluator::print(std::ostream &os) { std::cout << "--- RooFit BatchMode evaluation ---\n"; - std::vector widths{9, 37, 20, 9, 10, 20}; + std::vector widths{9, 37, 20, 9, 10, 4, 20}; auto printElement = [&](int iCol, auto const &t) { const char separator = ' '; @@ -755,7 +801,8 @@ void Evaluator::print(std::ostream &os) printElement(2, "Class"); printElement(3, "Size"); printElement(4, "From Data"); - printElement(5, "1st value"); + printElement(5, "Log"); + printElement(6, "1st value"); std::cout << "\n"; printHorizontalRow(); @@ -772,7 +819,8 @@ void Evaluator::print(std::ostream &os) printElement(2, node->ClassName()); printElement(3, nodeInfo.outputSize); printElement(4, nodeInfo.fromArrayInput); - printElement(5, span[0]); + printElement(5, _evalContextCPU.outputsLogarithm(node)); + printElement(6, span[0]); std::cout << "\n"; } diff --git a/roofit/roofitcore/src/RooNLLVarNew.cxx b/roofit/roofitcore/src/RooNLLVarNew.cxx index 4c7c8810bca66..b45af93cf9403 100644 --- a/roofit/roofitcore/src/RooNLLVarNew.cxx +++ b/roofit/roofitcore/src/RooNLLVarNew.cxx @@ -220,6 +220,7 @@ RooNLLVarNew::RooNLLVarNew(const RooNLLVarNew &other, const char *name) _funcMode{other._funcMode}, _chi2ErrorType{other._chi2ErrorType}, _simCount{other._simCount}, + _logSimCount{other._logSimCount}, _prefix{other._prefix}, _binw{other._binw} { @@ -475,7 +476,7 @@ void RooNLLVarNew::finalizeResult(RooFit::EvalContext &ctx, ROOT::Math::KahanSum // If we do bin-by bin offsetting, we don't do this because it cancels out. // The correction is specific to NLL; it has no meaning for chi2. if (_statistic == Statistic::NLL && !_doBinOffset && _simCount > 1) { - result += weightSum * std::log(static_cast(_simCount)); + result += weightSum * _logSimCount; } // Check if value offset flag is set. diff --git a/roofit/roofitcore/src/RooNormalizedPdf.cxx b/roofit/roofitcore/src/RooNormalizedPdf.cxx index 1505c0e811548..580575fc0591a 100644 --- a/roofit/roofitcore/src/RooNormalizedPdf.cxx +++ b/roofit/roofitcore/src/RooNormalizedPdf.cxx @@ -31,6 +31,16 @@ void RooNormalizedPdf::doEval(RooFit::EvalContext &ctx) const auto nums = ctx.at(_pdf); auto integralSpan = ctx.at(_normIntegral); + if (ctx.outputsLogarithm(this)) { + auto output = ctx.output(); + for (std::size_t i = 0; i < output.size(); ++i) { + double num = nums.size() > 1 ? nums[i] : nums[0]; + double den = integralSpan.size() > 1 ? integralSpan[i] : integralSpan[0]; + output[i] = num - den; + } + return; + } + // We use the extraArgs as output parameter to count evaluation errors. _evalErrorCounts = {}; diff --git a/roofit/roofitcore/src/RooRealIntegral.cxx b/roofit/roofitcore/src/RooRealIntegral.cxx index 57efcb422b451..0d3a8dec2e9ca 100644 --- a/roofit/roofitcore/src/RooRealIntegral.cxx +++ b/roofit/roofitcore/src/RooRealIntegral.cxx @@ -43,6 +43,7 @@ integration is performed in the various implementations of the RooAbsIntegrator #include #include #include +#include #include #include @@ -1120,6 +1121,17 @@ Int_t RooRealIntegral::getCacheAllNumeric() return _cacheAllNDim; } +void RooRealIntegral::doEval(RooFit::EvalContext &ctx) const +{ + RooAbsReal::doEval(ctx); + if (ctx.outputsLogarithm(this)) { + auto output = ctx.output(); + for (std::size_t i = 0; i < output.size(); ++i) { + output[i] = std::log(output[i]); + } + } +} + std::unique_ptr RooRealIntegral::compileForNormSet(RooArgSet const &normSet, RooFit::Detail::CompileContext &ctx) const { From 8aff41ab68becea2a5881a0bb3ef2890d74dbebe Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 17 Nov 2026 15:54:06 +0000 Subject: [PATCH 2/6] [RF] Evaluate logarithms of products and pdfs in likelihoods Generalize the logarithm optimization from the constraint terms to all nodes whose only value client takes the logarithm anyway, which are the RooConstraintSum and the unbinned RooNLLVarNew. A RooFixedProdPdf or RooNormalizedPdf that outputs its logarithm requests the same from its servers. A class tells with the new RooAbsArg::canUseLogarithmOf() from which servers it accepts logarithms. Evaluator::setupLogEvaluation() then requests the logarithm from a server that has no other client and whose values are not given as input. With the cpu backend, the likelihood of a product with many factors is then finite even if the product underflows. The factors that can't provide their logarithm are multiplied, and if a partial product left the range of normal doubles, the product is recomputed with its binary exponent kept separate. Avoiding exp() and log() speeds up Gaussian fits with 1M events by 2.6x for one Gaussian and 2.2x for a product of two. Products of factors without logarithms get about 2% slower. The logarithms are only requested for nodes evaluated on the CPU, so the cuda backend can still underflow. With the codegen backend, only the likelihood value benefits, not the gradient. The RooRealIntegral doesn't output its logarithm anymore, because a negative integral then lost the badness packed into the NaN. The tests that compared the cpu backend exactly with the RooAbsL classes now use tolerances: the likelihoods differ by 1 ULP, the fitted parameters by up to 1e-10. This fixes the reproducer of #24202 with the cpu backend. Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit 6bf1cd05ab14fbc57366b49c17cf397c99c61280) --- roofit/batchcompute/res/RooBatchCompute.h | 9 +- roofit/batchcompute/src/ComputeFunctions.cxx | 79 ++++++++++++++ roofit/batchcompute/src/RooBatchCompute.cu | 23 +++- roofit/batchcompute/src/RooBatchCompute.cxx | 47 ++++++-- roofit/roofitcore/inc/RooAbsArg.h | 5 + roofit/roofitcore/inc/RooConstraintSum.h | 1 + .../inc/RooFit/Detail/RooNLLVarNew.h | 1 + .../inc/RooFit/Detail/RooNormalizedPdf.h | 7 +- roofit/roofitcore/inc/RooFit/Evaluator.h | 1 + roofit/roofitcore/inc/RooProdPdf.h | 2 + roofit/roofitcore/inc/RooRealIntegral.h | 3 - roofit/roofitcore/src/RooFit/Evaluator.cxx | 74 +++++-------- roofit/roofitcore/src/RooNLLVarNew.cxx | 9 +- roofit/roofitcore/src/RooNormalizedPdf.cxx | 60 +++++++++-- roofit/roofitcore/src/RooProdPdf.cxx | 14 +++ roofit/roofitcore/src/RooRealIntegral.cxx | 12 --- .../testLikelihoodGradientJob.cxx | 12 ++- .../test/TestStatistics/testLikelihoodJob.cxx | 2 +- .../TestStatistics/testLikelihoodSerial.cxx | 2 +- .../test/TestStatistics/testRooRealL.cxx | 12 ++- roofit/roofitcore/test/testNaNPacker.cxx | 30 ++++++ .../roofitcore/test/testRooBatchCompute.cxx | 100 ++++++++++++++++++ roofit/roofitcore/test/testRooEvaluator.cxx | 29 +++++ roofit/roofitcore/test/testRooProdPdf.cxx | 86 +++++++++++++++ 24 files changed, 524 insertions(+), 96 deletions(-) diff --git a/roofit/batchcompute/res/RooBatchCompute.h b/roofit/batchcompute/res/RooBatchCompute.h index 8297967e98980..8cc6bbeb4786e 100644 --- a/roofit/batchcompute/res/RooBatchCompute.h +++ b/roofit/batchcompute/res/RooBatchCompute.h @@ -110,6 +110,7 @@ enum Computer { Polynomial, Power, ProdPdf, + LogProdPdf, Ratio, TruthModelExpBasis, TruthModelSinBasis, @@ -191,8 +192,10 @@ class RooBatchComputeInterface { virtual void compute(Config const &cfg, Computer, std::span output, VarSpan, ArgSpan) = 0; virtual double reduceSum(Config const &cfg, InputArr input, size_t n) = 0; + /// Reduce the probabilities to the negative log-likelihood. With + /// `logProbas`, the probabilities are given as logarithms. virtual ReduceNLLOutput reduceNLL(Config const &cfg, std::span probas, std::span weights, - std::span offsetProbas) = 0; + std::span offsetProbas, bool logProbas) = 0; virtual Architecture architecture() const = 0; virtual std::string architectureName() const = 0; @@ -247,10 +250,10 @@ inline double reduceSum(Config cfg, InputArr input, size_t n) } inline ReduceNLLOutput reduceNLL(Config cfg, std::span probas, std::span weights, - std::span offsetProbas) + std::span offsetProbas, bool logProbas = false) { auto dispatch = cfg.useCuda() ? dispatchCUDA : dispatchCPU; - return dispatch->reduceNLL(cfg, probas, weights, offsetProbas); + return dispatch->reduceNLL(cfg, probas, weights, offsetProbas, logProbas); } std::string getBatchComputeChoice(); diff --git a/roofit/batchcompute/src/ComputeFunctions.cxx b/roofit/batchcompute/src/ComputeFunctions.cxx index febc7e71b0911..1f313fa4c0748 100644 --- a/roofit/batchcompute/src/ComputeFunctions.cxx +++ b/roofit/batchcompute/src/ComputeFunctions.cxx @@ -31,6 +31,8 @@ of performance, maximum memory coalescing. For more details, see #include +#include +#include #include #ifdef __CUDACC__ @@ -797,6 +799,82 @@ __rooglobal__ void computeProdPdf(Batches &batches) } } +/// Logarithm of the product of the factors [begin, end) for event `i`. The +/// binary exponent is kept separate, so that the product can't underflow. +__roodevice__ inline double logOfProduct(Batches const &batches, std::size_t begin, std::size_t end, std::size_t i) +{ + constexpr double ln2 = 0.693147180559945309417232121458176568; + // The mantissa stays within [2^-256, 2^256] and the factors are brought + // into [2^-512, 2^512], so their product is always a normal double. + double mantissa = 1.0; + int exponent = 0; + for (std::size_t k = begin; k < end; ++k) { + double factor = batches.args[k][i]; + int e = 0; + if (std::abs(factor) < 0x1p-512 || std::abs(factor) > 0x1p512) { + factor = std::frexp(factor, &e); + exponent += e; + } + mantissa *= factor; + if (std::abs(mantissa) < 0x1p-256 || std::abs(mantissa) > 0x1p256) { + mantissa = std::frexp(mantissa, &e); + exponent += e; + } + } + if (std::isnan(mantissa)) { + return mantissa; // with the payload of the NaN factor + } + // Pack a negative product into a NaN, like computeNormalizedPdf does. + return mantissa < 0. ? RooNaNPacker::packFloatIntoNaN(-std::ldexp(mantissa, exponent)) + : std::log(mantissa) + exponent * ln2; +} + +/// Logarithm of a product. The first `extra[0]` factors are given as +/// logarithms and are summed, the others are multiplied. +__rooglobal__ void computeLogProdPdf(Batches &batches) +{ + const std::size_t nLogFactors = batches.extra[0]; + const std::size_t nFactors = batches.nBatches; + for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { + batches.output[i] = 0.0; + } + for (std::size_t k = 0; k < nLogFactors; ++k) { + for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { + batches.output[i] += batches.args[k][i]; + } + } + if (nLogFactors == nFactors) { + return; + } +#ifdef __CUDACC__ + for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { + batches.output[i] += logOfProduct(batches, nLogFactors, nFactors, i); + } +#else + // Multiply in a vectorizable loop, keeping track of the smallest partial + // product. If that is a normal number and the product didn't overflow, + // no precision was lost. Otherwise, the event is recomputed with + // logOfProduct(). + double prod[bufferSize]; + double minProd[bufferSize]; + std::fill_n(prod, batches.nEvents, 1.0); + std::fill_n(minProd, batches.nEvents, 1.0); + for (std::size_t k = nLogFactors; k < nFactors; ++k) { + for (std::size_t i = 0; i < batches.nEvents; ++i) { + prod[i] *= batches.args[k][i]; + minProd[i] = std::min(minProd[i], prod[i]); + } + } + for (std::size_t i = 0; i < batches.nEvents; ++i) { + if (minProd[i] >= std::numeric_limits::min() && prod[i] <= std::numeric_limits::max()) { + batches.output[i] += std::log(prod[i]); + } else { + batches.output[i] += logOfProduct(batches, nLogFactors, nFactors, i); + } + } +#endif +} + __rooglobal__ void computeRatio(Batches &batches) { for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { @@ -969,6 +1047,7 @@ std::vector getFunctions() computePolynomial, computePower, computeProdPdf, + computeLogProdPdf, computeRatio, computeTruthModelExpBasis, computeTruthModelSinBasis, diff --git a/roofit/batchcompute/src/RooBatchCompute.cu b/roofit/batchcompute/src/RooBatchCompute.cu index b277c943c157c..4d76088c0e4b8 100644 --- a/roofit/batchcompute/src/RooBatchCompute.cu +++ b/roofit/batchcompute/src/RooBatchCompute.cu @@ -309,7 +309,8 @@ public: /// Return the sum of an input array double reduceSum(RooBatchCompute::Config const &cfg, InputArr input, size_t n) override; ReduceNLLOutput reduceNLL(RooBatchCompute::Config const &cfg, std::span probas, - std::span weights, std::span offsetProbas) override; + std::span weights, std::span offsetProbas, + bool logProbas) override; std::unique_ptr createBufferManager() const override; @@ -405,9 +406,10 @@ __global__ void kahanSum(const double *__restrict__ input, const double *__restr /// into a "badness" value that the host can pack into a NaN for the error /// recovery in the minimizer. The `stats` output has the layout /// [badness, nNonPositive, nNaN, nInfinite] and must be zero-initialized. +/// With `logProbas`, the probabilities are given as logarithms. __global__ void nllSumKernel(const double *__restrict__ probas, const double *__restrict__ weights, const double *__restrict__ offsetProbas, size_t nProbas, double scalarProba, - size_t nWeights, double *__restrict__ result, double *__restrict__ stats) + bool logProbas, size_t nWeights, double *__restrict__ result, double *__restrict__ stats) { int thIdx = threadIdx.x; int gthIdx = thIdx + blockIdx.x * blockSize; @@ -433,7 +435,17 @@ __global__ void nllSumKernel(const double *__restrict__ probas, const double *__ } const double proba = nProbas == 1 ? scalarProba : probas[i]; double term; - if (proba <= 0.0) { + if (logProbas) { + if (std::isnan(proba)) { + ++nNaN; + badness += RooNaNPacker::unpackNaN(proba); + } else if (proba == -INFINITY) { + ++nNonPositive; + } else if (std::isinf(proba)) { + ++nInfinite; + } + term = proba; + } else if (proba <= 0.0) { ++nNonPositive; badness += -proba; term = std::log(proba); @@ -496,7 +508,8 @@ double RooBatchComputeClass::reduceSum(RooBatchCompute::Config const &cfg, Input } ReduceNLLOutput RooBatchComputeClass::reduceNLL(RooBatchCompute::Config const &cfg, std::span probas, - std::span weights, std::span offsetProbas) + std::span weights, std::span offsetProbas, + bool logProbas) { ReduceNLLOutput out; if (probas.empty()) { @@ -528,7 +541,7 @@ ReduceNLLOutput RooBatchComputeClass::reduceNLL(RooBatchCompute::Config const &c nllSumKernel<<>>( probas.data(), weights.data(), offsetProbas.empty() ? nullptr : offsetProbas.data(), probas.size(), - probas.size() == 1 ? probas[0] : 0.0, weights.size(), devOut + 6, devOut + 2); + probas.size() == 1 ? probas[0] : 0.0, logProbas, weights.size(), devOut + 6, devOut + 2); kahanSum<<<1, blockSize, shMemSize, stream>>>(devOut + 6, devOut + 6 + gridSize, gridSize, devOut, 0); diff --git a/roofit/batchcompute/src/RooBatchCompute.cxx b/roofit/batchcompute/src/RooBatchCompute.cxx index 8201dafeb4c2c..db3dbbf12247d 100644 --- a/roofit/batchcompute/src/RooBatchCompute.cxx +++ b/roofit/batchcompute/src/RooBatchCompute.cxx @@ -34,6 +34,7 @@ This file contains the code for cpu computations using the RooBatchCompute libra #include #include +#include #include #include #include @@ -209,7 +210,7 @@ class RooBatchComputeClass : public RooBatchComputeInterface { void compute(Config const &, Computer computer, std::span output, VarSpan vars, ArgSpan extraArgs) override; double reduceSum(Config const &, InputArr input, size_t n) override; ReduceNLLOutput reduceNLL(Config const &, std::span probas, std::span weights, - std::span offsetProbas) override; + std::span offsetProbas, bool logProbas) override; std::unique_ptr createBufferManager() const override; @@ -292,6 +293,23 @@ inline std::pair getLog(double prob, ReduceNLLOutput &out) return {std::log(prob), 0.0}; } +/// Like getLog(), but for a probability that is already given as logarithm. +inline std::pair checkLog(double logProb, ReduceNLLOutput &out) +{ + if (std::isnan(logProb)) { + out.nNaNValues++; + return {logProb, RooNaNPacker::unpackNaN(logProb)}; + } + + if (logProb == -std::numeric_limits::infinity()) { + out.nNonPositiveValues++; + } else if (std::isinf(logProb)) { + out.nInfiniteValues++; + } + + return {logProb, 0.0}; +} + } // namespace double RooBatchComputeClass::reduceSum(Config const &cfg, InputArr input, size_t n) @@ -328,6 +346,9 @@ struct NLLPartialResult { ReduceNLLOutput counters; }; +/// With `LogProbas`, the probabilities are given as logarithms. It's a +/// template parameter, so that no logarithms are evaluated speculatively. +template void reduceNLLRange(std::span probas, std::span weights, std::span offsetProbas, std::size_t begin, std::size_t end, NLLPartialResult &result) { @@ -336,7 +357,8 @@ void reduceNLLRange(std::span probas, std::span weig if (0. == weights[i]) continue; - std::pair logOut = getLog(probas.size() == 1 ? probas[0] : probas[i], result.counters); + const double proba = probas.size() == 1 ? probas[0] : probas[i]; + std::pair logOut = LogProbas ? checkLog(proba, result.counters) : getLog(proba, result.counters); double term = logOut.first; result.badness += logOut.second; @@ -350,10 +372,22 @@ void reduceNLLRange(std::span probas, std::span weig } } +void reduceNLLRange(std::span probas, std::span weights, + std::span offsetProbas, bool logProbas, std::size_t begin, std::size_t end, + NLLPartialResult &result) +{ + if (logProbas) { + reduceNLLRange(probas, weights, offsetProbas, begin, end, result); + } else { + reduceNLLRange(probas, weights, offsetProbas, begin, end, result); + } +} + } // namespace ReduceNLLOutput RooBatchComputeClass::reduceNLL(Config const &cfg, std::span probas, - std::span weights, std::span offsetProbas) + std::span weights, std::span offsetProbas, + bool logProbas) { const std::size_t n = weights.size(); NLLPartialResult result; @@ -364,7 +398,8 @@ ReduceNLLOutput RooBatchComputeClass::reduceNLL(Config const &cfg, std::span partials(nChunks); parallelForChunks(cfg.nThreads(), n, [&](std::size_t iChunk) { const std::size_t begin = chunkBegin(iChunk); - reduceNLLRange(probas, weights, offsetProbas, begin, begin + chunkSize(iChunk, n), partials[iChunk]); + reduceNLLRange(probas, weights, offsetProbas, logProbas, begin, begin + chunkSize(iChunk, n), + partials[iChunk]); }); // Combine the partial results in fixed chunk order, so that the result // doesn't depend on the number of threads. @@ -376,11 +411,11 @@ ReduceNLLOutput RooBatchComputeClass::reduceNLL(Config const &cfg, std::span compileForNormSet(RooArgSet const &normSet, RooFit::Detail::CompileContext & ctx) const override; diff --git a/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h b/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h index 6e0f17fb3b4fa..19ee5cebfedb6 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h +++ b/roofit/roofitcore/inc/RooFit/Detail/RooNLLVarNew.h @@ -62,6 +62,7 @@ class RooNLLVarNew : public RooAbsReal { void doEval(RooFit::EvalContext &) const override; bool canComputeBatchWithCuda() const override { return _statistic == Statistic::NLL && !_binnedL; } bool isReducerNode() const override { return true; } + bool canUseLogarithmOf(RooAbsArg const &server, bool outputsLogarithm) const override; void setPrefix(std::string const &prefix); diff --git a/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h b/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h index 2689b6ca6b562..99cbeee768a80 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h +++ b/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h @@ -79,7 +79,12 @@ class RooNormalizedPdf : public RooAbsPdf { return _pdf->createExpectedEventsFunc(&_normSet); } - bool canOutputLogarithm() const override { return true; } + // Only worth it if the pdf can output its logarithm, avoiding exp() and log(). + bool canOutputLogarithm() const override { return _pdf->canOutputLogarithm(); } + bool canUseLogarithmOf(RooAbsArg const &server, bool outputsLogarithm) const override + { + return outputsLogarithm && &server == &*_pdf; + } bool canComputeBatchWithCuda() const override { return true; } RooAbsPdf const &pdf() const { return *_pdf; } diff --git a/roofit/roofitcore/inc/RooFit/Evaluator.h b/roofit/roofitcore/inc/RooFit/Evaluator.h index 262522407e12f..65a054479ed9a 100644 --- a/roofit/roofitcore/inc/RooFit/Evaluator.h +++ b/roofit/roofitcore/inc/RooFit/Evaluator.h @@ -63,6 +63,7 @@ class Evaluator { void computeCPUNode(const RooAbsArg *node, NodeInfo &info); void setOperMode(RooAbsArg *arg, RooAbsArg::OperMode opMode); void syncDataTokens(); + void setupLogEvaluation(); void updateOutputSizes(); std::unique_ptr _bufferManager; diff --git a/roofit/roofitcore/inc/RooProdPdf.h b/roofit/roofitcore/inc/RooProdPdf.h index 4ef2ec50c86ae..2e6d3866e3bca 100644 --- a/roofit/roofitcore/inc/RooProdPdf.h +++ b/roofit/roofitcore/inc/RooProdPdf.h @@ -217,6 +217,8 @@ class RooFixedProdPdf : public RooAbsPdf { inline bool selfNormalized() const override { return true; } inline bool canComputeBatchWithCuda() const override { return true; } + inline bool canOutputLogarithm() const override { return !_isRearranged; } + inline bool canUseLogarithmOf(RooAbsArg const &, bool outputsLogarithm) const override { return outputsLogarithm; } void doEval(RooFit::EvalContext &ctx) const override; diff --git a/roofit/roofitcore/inc/RooRealIntegral.h b/roofit/roofitcore/inc/RooRealIntegral.h index d84df3b73053b..04e1b9d9c8573 100644 --- a/roofit/roofitcore/inc/RooRealIntegral.h +++ b/roofit/roofitcore/inc/RooRealIntegral.h @@ -103,9 +103,6 @@ class RooRealIntegral : public RooAbsReal { double evaluate() const override ; bool isValidReal(double value, bool printError=false) const override ; - void doEval(RooFit::EvalContext &) const override; - bool canOutputLogarithm() const override { return true; } - bool redirectServersHook(const RooAbsCollection& newServerList, bool mustReplaceAll, bool nameChange, bool isRecursive) override ; diff --git a/roofit/roofitcore/src/RooFit/Evaluator.cxx b/roofit/roofitcore/src/RooFit/Evaluator.cxx index c098950af5132..320e3bcfb6208 100644 --- a/roofit/roofitcore/src/RooFit/Evaluator.cxx +++ b/roofit/roofitcore/src/RooFit/Evaluator.cxx @@ -33,8 +33,6 @@ RooAbsPdf::fitTo() is called and gets destroyed when the fitting ends. #include #include #include -#include -#include #include #include #include @@ -223,50 +221,6 @@ Evaluator::Evaluator(const RooAbsReal &absReal, bool useGPU) syncDataTokens(); - // This is an optimization for Gaussian constraint terms, of which there are - // many in big fits. To Avoid taking the the exponential and then the - // logarithm for the constraint term, we request that all normalized pdfs - // that have only the RooConstraintSum as a value client are evaluating - // their logarithms, which the RooConstrantSum::doEval() then assumes. - // Note that we don't need to consider the GPU evaluation path here, because - // scalar terms like the constraints are always evaluated on the CPU. - for (NodeInfo &info : _nodes) { - if (!dynamic_cast(info.absArg)) { - continue; - } - for (RooAbsArg *server : info.absArg->servers()) { - if (!server->canOutputLogarithm()) { - continue; - } - // If the pdf has clients other than the RooConstraintSum, we can't do - // the log optimization. - if (server->clients().size() != 1) { - continue; - } - auto &outputsLogarithm = _evalContextCPU._outputsLogarithm; - if (auto normPdf = dynamic_cast(server)) { - // If this is a normalized pdf, we also request the log of the pdf - // value and normalization integral. - // Disable the optimization if the computation graph doesn't have - // the expected structure, for safety. The unnormalized pdf has two - // clients: the integral, and the RooNormalizedPdf. - bool const canOptimizePdf = normPdf->pdf().canOutputLogarithm() && normPdf->pdf().clients().size() == 2; - bool const canOptimizeNormIntegral = - normPdf->normIntegral().canOutputLogarithm() && normPdf->normIntegral().clients().size() == 1; - // If the servers can't be optimized, no gain in optimizing the - // normalized pdf. - if (!canOptimizePdf || !canOptimizeNormIntegral) { - continue; - } - // Only now set the flags, such that the log optimization for - // the normalized pdf and its servers is enabled atomically. - outputsLogarithm[normPdf->pdf().dataToken()] = true; - outputsLogarithm[normPdf->normIntegral().dataToken()] = true; - } - outputsLogarithm[server->dataToken()] = true; - } - } - if (_useGPU) { // Create the single CUDA stream on which all GPU computations and data // transfers of this Evaluator are enqueued. The graph is evaluated in @@ -279,6 +233,8 @@ Evaluator::Evaluator(const RooAbsReal &absReal, bool useGPU) _evalContextCUDA.setConfig(info.absArg, cfg); } } + + setupLogEvaluation(); } /// If there are servers with the same name that got de-duplicated in the @@ -300,6 +256,29 @@ void Evaluator::syncDataTokens() } } +/// Request nodes to output their logarithms if their only value client takes +/// the logarithm anyway, like the RooNLLVarNew and RooConstraintSum do. This +/// avoids exp() -> log() roundtrips, e.g. for Gaussian constraints, and the +/// underflow of products with many factors. Nodes that output their +/// logarithms can request the same from their servers, e.g. the factors of a +/// product. Only implemented for nodes evaluated on the CPU. +void Evaluator::setupLogEvaluation() +{ + // Visit clients before their servers, because whether a client can use a + // logarithm can depend on its own output mode. + for (auto iter = _nodes.rbegin(); iter != _nodes.rend(); ++iter) { + NodeInfo &info = *iter; + bool outputsLogarithm = false; + if (info.clientInfos.size() == 1 && !info.fromArrayInput && !info.computeInGPU && + info.absArg->canOutputLogarithm()) { + NodeInfo const &client = *info.clientInfos[0]; + outputsLogarithm = !client.computeInGPU && client.absArg->canUseLogarithmOf( + *info.absArg, _evalContextCPU._outputsLogarithm[client.iNode]); + } + _evalContextCPU._outputsLogarithm[info.iNode] = outputsLogarithm; + } +} + void Evaluator::setInput(std::string const &name, std::span inputArray, bool isOnDevice) { if (isOnDevice && !_useGPU) { @@ -390,6 +369,9 @@ void Evaluator::updateOutputSizes() markGPUNodes(); } + // Which nodes are inputs or evaluated on the GPU might have changed. + setupLogEvaluation(); + _needToUpdateOutputSizes = false; } diff --git a/roofit/roofitcore/src/RooNLLVarNew.cxx b/roofit/roofitcore/src/RooNLLVarNew.cxx index b45af93cf9403..b20acbfc1194b 100644 --- a/roofit/roofitcore/src/RooNLLVarNew.cxx +++ b/roofit/roofitcore/src/RooNLLVarNew.cxx @@ -373,6 +373,12 @@ void RooNLLVarNew::doEvalChi2(RooFit::EvalContext &ctx, std::span finalizeResult(ctx, result, sumWeightKahanSum.Sum()); } +/// The unbinned likelihood can directly take the logarithms of the pdf values. +bool RooNLLVarNew::canUseLogarithmOf(RooAbsArg const &server, bool /*outputsLogarithm*/) const +{ + return _statistic == Statistic::NLL && !_binnedL && &server == &*_func; +} + void RooNLLVarNew::doEval(RooFit::EvalContext &ctx) const { std::span weights = ctx.at(_weightVar); @@ -397,7 +403,8 @@ void RooNLLVarNew::doEval(RooFit::EvalContext &ctx) const } auto nllOut = RooBatchCompute::reduceNLL(config, probas, _weightSquared ? weightsSumW2 : weights, - _doBinOffset ? ctx.at(*_offsetPdf) : std::span{}); + _doBinOffset ? ctx.at(*_offsetPdf) : std::span{}, + ctx.outputsLogarithm(&*_func)); if (nllOut.nInfiniteValues > 0) { oocoutW(&*_func, Eval) << "RooAbsPdf::getLogVal(" << _func->GetName() diff --git a/roofit/roofitcore/src/RooNormalizedPdf.cxx b/roofit/roofitcore/src/RooNormalizedPdf.cxx index 580575fc0591a..86cba72c1e199 100644 --- a/roofit/roofitcore/src/RooNormalizedPdf.cxx +++ b/roofit/roofitcore/src/RooNormalizedPdf.cxx @@ -14,8 +14,11 @@ #include "RooBatchCompute.h" #include "RooFitImplHelpers.h" +#include "RooNaNPacker.h" #include +#include +#include /** * \class RooNormalizedPdf @@ -24,6 +27,43 @@ * normalization set into a new self-normalized pdf. */ +namespace { + +/// Logarithm of the normalized pdf, with the same evaluation errors as the +/// NormalizedPdf compute function. The input modes are template parameters, +/// so that no logarithms are evaluated speculatively. +template +void evalLogNormalized(std::span output, std::span nums, std::span dens, + std::array &errorCounts) +{ + constexpr double negInf = -std::numeric_limits::infinity(); + const double scalarLogDen = ScalarDen ? std::log(dens[0]) : 0.; + for (std::size_t i = 0; i < output.size(); ++i) { + const double num = nums.size() > 1 ? nums[i] : nums[0]; + const double den = ScalarDen ? dens[0] : dens[i]; + const bool numIsZero = NumIsLog ? num == negInf : num == 0.; + const bool numIsNegative = !NumIsLog && num < 0.; + if (den < 0. || (den == 0. && !numIsZero)) { + output[i] = RooNaNPacker::packFloatIntoNaN(-den + (numIsNegative ? -num : 0.)); + ++errorCounts[0]; + } else if (numIsNegative) { + output[i] = RooNaNPacker::packFloatIntoNaN(-num); + ++errorCounts[1]; + } else if (std::isnan(num)) { + output[i] = num; + ++errorCounts[2]; + } else if (numIsZero) { + output[i] = negInf; + } else { + const double logNum = NumIsLog ? num : std::log(num); + const double logDen = ScalarDen ? scalarLogDen : std::log(den); + output[i] = logNum - logDen; + } + } +} + +} // namespace + namespace RooFit::Detail { void RooNormalizedPdf::doEval(RooFit::EvalContext &ctx) const @@ -31,19 +71,21 @@ void RooNormalizedPdf::doEval(RooFit::EvalContext &ctx) const auto nums = ctx.at(_pdf); auto integralSpan = ctx.at(_normIntegral); + // We use the extraArgs as output parameter to count evaluation errors. + _evalErrorCounts = {}; + if (ctx.outputsLogarithm(this)) { - auto output = ctx.output(); - for (std::size_t i = 0; i < output.size(); ++i) { - double num = nums.size() > 1 ? nums[i] : nums[0]; - double den = integralSpan.size() > 1 ? integralSpan[i] : integralSpan[0]; - output[i] = num - den; - } + // The pdf might provide its logarithm already. + const bool numIsLog = ctx.outputsLogarithm(&*_pdf); + const bool scalarDen = integralSpan.size() == 1; + auto evalLog = numIsLog ? (scalarDen ? evalLogNormalized : evalLogNormalized) + : (scalarDen ? evalLogNormalized : evalLogNormalized); + evalLog(ctx.output(), nums, integralSpan, _evalErrorCounts); + // The logarithm is only requested in CPU evaluation. + logEvalErrorCounts(); return; } - // We use the extraArgs as output parameter to count evaluation errors. - _evalErrorCounts = {}; - auto config = ctx.config(this); RooBatchCompute::compute(config, RooBatchCompute::NormalizedPdf, ctx.output(), {nums, integralSpan}, _evalErrorCounts); diff --git a/roofit/roofitcore/src/RooProdPdf.cxx b/roofit/roofitcore/src/RooProdPdf.cxx index 1ecc11461e0c1..b49d2e96db503 100644 --- a/roofit/roofitcore/src/RooProdPdf.cxx +++ b/roofit/roofitcore/src/RooProdPdf.cxx @@ -2205,6 +2205,20 @@ void RooFixedProdPdf::doEval(RooFit::EvalContext &ctx) const } std::vector> factors; factors.reserve(partList()->size()); + if (ctx.outputsLogarithm(this)) { + // The factors that provide their logarithms go first. + for (const RooAbsArg *arg : *partList()) { + if (ctx.outputsLogarithm(arg)) + factors.push_back(ctx.at(arg)); + } + std::array nLogFactors{static_cast(factors.size())}; + for (const RooAbsArg *arg : *partList()) { + if (!ctx.outputsLogarithm(arg)) + factors.push_back(ctx.at(arg)); + } + RooBatchCompute::compute(ctx.config(this), RooBatchCompute::LogProdPdf, ctx.output(), factors, nLogFactors); + return; + } for (const RooAbsArg *arg : *partList()) { auto span = ctx.at(arg); factors.push_back(span); diff --git a/roofit/roofitcore/src/RooRealIntegral.cxx b/roofit/roofitcore/src/RooRealIntegral.cxx index 0d3a8dec2e9ca..57efcb422b451 100644 --- a/roofit/roofitcore/src/RooRealIntegral.cxx +++ b/roofit/roofitcore/src/RooRealIntegral.cxx @@ -43,7 +43,6 @@ integration is performed in the various implementations of the RooAbsIntegrator #include #include #include -#include #include #include @@ -1121,17 +1120,6 @@ Int_t RooRealIntegral::getCacheAllNumeric() return _cacheAllNDim; } -void RooRealIntegral::doEval(RooFit::EvalContext &ctx) const -{ - RooAbsReal::doEval(ctx); - if (ctx.outputsLogarithm(this)) { - auto output = ctx.output(); - for (std::size_t i = 0; i < output.size(); ++i) { - output[i] = std::log(output[i]); - } - } -} - std::unique_ptr RooRealIntegral::compileForNormSet(RooArgSet const &normSet, RooFit::Detail::CompileContext &ctx) const { diff --git a/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx b/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx index db0641eba7977..3e6919cddda85 100644 --- a/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx +++ b/roofit/roofitcore/test/TestStatistics/testLikelihoodGradientJob.cxx @@ -188,10 +188,14 @@ TEST_P(LikelihoodGradientJobTest, Gaussian1D) double mu1 = mu->getVal(); double muerr1 = mu->getError(); - EXPECT_EQ(minNll0, minNll1); - EXPECT_EQ(mu0, mu1); - EXPECT_EQ(muerr0, muerr1); - EXPECT_EQ(edm0, edm1); + // The cpu backend and RooUnbinnedL are different implementations, so the + // minimizations don't follow the same path exactly. +#define EXPECT_NEAR_REL(a, b, c) EXPECT_NEAR(a, b, std::abs((a) * (c))) + EXPECT_NEAR_REL(minNll0, minNll1, 1e-7); + EXPECT_NEAR_REL(mu0, mu1, 1e-7); + EXPECT_NEAR_REL(muerr0, muerr1, 1e-7); + EXPECT_NEAR_REL(edm0, edm1, 1e-4); +#undef EXPECT_NEAR_REL } TEST(LikelihoodGradientJob, RepeatMigrad) diff --git a/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx b/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx index 4099e88d6dba3..e6673ca9defa1 100644 --- a/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx +++ b/roofit/roofitcore/test/TestStatistics/testLikelihoodJob.cxx @@ -399,7 +399,7 @@ TEST_F(LikelihoodJobTest, SimUnbinnedNonExtended) nll_ts->evaluate(); auto nll1 = nll_ts->getResult(); - EXPECT_EQ(nll0, nll1.Sum()); + EXPECT_DOUBLE_EQ(nll0, nll1.Sum()); } class LikelihoodJobSimBinnedConstrainedTest : public LikelihoodJobTest { diff --git a/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx b/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx index 8751767524352..ff55cefac9987 100644 --- a/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx +++ b/roofit/roofitcore/test/TestStatistics/testLikelihoodSerial.cxx @@ -324,7 +324,7 @@ TEST_F(LikelihoodSerialTest, SimUnbinnedNonExtended) nll_ts->evaluate(); auto nll1 = nll_ts->getResult(); - EXPECT_EQ(nll0, nll1.Sum()); + EXPECT_DOUBLE_EQ(nll0, nll1.Sum()); } class LikelihoodSerialSimBinnedConstrainedTest : public LikelihoodSerialTest { diff --git a/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx b/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx index 2e6099aba550a..fc734580ac24e 100644 --- a/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx +++ b/roofit/roofitcore/test/TestStatistics/testRooRealL.cxx @@ -354,10 +354,14 @@ TEST_P(RealLVsMPFE, minimize) double mu1 = mu->getVal(); double muerr1 = mu->getError(); - EXPECT_EQ(minNll0, minNll1); - EXPECT_EQ(mu0, mu1); - EXPECT_EQ(muerr0, muerr1); - EXPECT_EQ(edm0, edm1); + // The cpu backend and RooRealL are different implementations, so the + // minimizations don't follow the same path exactly. +#define EXPECT_NEAR_REL(a, b, c) EXPECT_NEAR(a, b, std::abs((a) * (c))) + EXPECT_NEAR_REL(minNll0, minNll1, 1e-7); + EXPECT_NEAR_REL(mu0, mu1, 1e-7); + EXPECT_NEAR_REL(muerr0, muerr1, 1e-7); + EXPECT_NEAR_REL(edm0, edm1, 1e-4); +#undef EXPECT_NEAR_REL } INSTANTIATE_TEST_SUITE_P(NworkersModeSeed, RealLVsMPFE, ::testing::Values(2, 3)); // random seed diff --git a/roofit/roofitcore/test/testNaNPacker.cxx b/roofit/roofitcore/test/testNaNPacker.cxx index aad258706186b..4e38933d07933 100644 --- a/roofit/roofitcore/test/testNaNPacker.cxx +++ b/roofit/roofitcore/test/testNaNPacker.cxx @@ -11,6 +11,9 @@ #include #include #include +#include + +#include #include "gtest_wrapper.h" @@ -299,6 +302,33 @@ TEST(RooNaNPacker, CudaEvalErrorParity) } #endif // ROOFIT_CUDA +/// A negative normalization integral is packed into the NaN as badness, also +/// if the pdf is evaluated in logarithmic scale. +TEST(RooNaNPacker, NegativeNormIntegralBadness) +{ + const auto prevErrorLoggingMode = RooAbsReal::evalErrorLoggingMode(); + RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::Ignore); + + RooWorkspace ws; + ws.factory("Gaussian::gauss(x[0, -10, 10], mean[0, -5, 5], sigma[1, -5, 5])"); + RooRealVar &x = *ws.var("x"); + RooDataSet data{"data", "data", x}; + for (double val : {0.1, -0.3, 1.5}) { + x.setVal(val); + data.add(x); + } + + std::unique_ptr nll{ws.pdf("gauss")->createNLL(data, RooFit::EvalBackend::Cpu())}; + ws.var("sigma")->setVal(-1.0); + const double val = nll->getVal(); + + // Each event contributes the negative integral sqrt(2 pi) * sigma. + EXPECT_TRUE(RooNaNPacker::isNaNWithPayload(val)); + EXPECT_NEAR(RooNaNPacker::unpackNaN(val), 3 * std::sqrt(2 * TMath::Pi()), 1e-5); + + RooAbsReal::setEvalErrorLoggingMode(prevErrorLoggingMode); +} + /// Make coefficients of RooAddPdf sum to more than 1. Fitter should recover from this. TEST(RooNaNPacker, FitAddPdf_DegenerateCoeff) { diff --git a/roofit/roofitcore/test/testRooBatchCompute.cxx b/roofit/roofitcore/test/testRooBatchCompute.cxx index a7addc1ec0f7b..10ac869a14f96 100644 --- a/roofit/roofitcore/test/testRooBatchCompute.cxx +++ b/roofit/roofitcore/test/testRooBatchCompute.cxx @@ -3,12 +3,16 @@ // Authors: Jonas Rembser, CERN 2026 #include +#include #include #include +#include +#include #include +#include #include #include @@ -151,3 +155,99 @@ TEST_F(RooBatchComputeMT, ReduceNLL) EXPECT_EQ(out.nNonPositiveValues, 2u) << "with " << nThreads << " threads"; } } + +/// Reducing the logarithms of the probabilities gives the same result and the +/// same evaluation error counters as reducing the probabilities. +TEST_F(RooBatchComputeMT, ReduceNLLFromLogarithms) +{ + std::vector probas = makeRandomVector(nEvents, 0.01, 1., 105); + std::vector weights = makeRandomVector(nEvents, 0.5, 2., 106); + auto logOf = [](std::vector const &values) { + std::vector out(values.size()); + std::transform(values.begin(), values.end(), out.begin(), [](double x) { return std::log(x); }); + return out; + }; + + for (int nThreads : {1, 4}) { + auto ref = RooBatchCompute::reduceNLL(makeConfig(nThreads), probas, weights, {}); + auto out = RooBatchCompute::reduceNLL(makeConfig(nThreads), logOf(probas), weights, {}, true); + EXPECT_NEAR(out.nllSum, ref.nllSum, 1e-12 * std::abs(ref.nllSum)) << "with " << nThreads << " threads"; + } + + probas[123] = 0.0; + probas[181818] = std::numeric_limits::quiet_NaN(); + for (int nThreads : {1, 4}) { + auto out = RooBatchCompute::reduceNLL(makeConfig(nThreads), logOf(probas), weights, {}, true); + EXPECT_EQ(out.nNonPositiveValues, 1u) << "with " << nThreads << " threads"; + EXPECT_EQ(out.nNaNValues, 1u) << "with " << nThreads << " threads"; + } +} + +/// The LogProdPdf computer sums the factors given as logarithms and +/// multiplies the others, without underflow for many small factors. +TEST_F(RooBatchComputeMT, ComputeLogProdPdf) +{ + constexpr std::size_t n = 1000; + constexpr std::size_t nLogFactors = 200; + constexpr std::size_t nFactors = 400; + + std::vector> factors; + std::vector ref(n, 0.0); + for (std::size_t k = 0; k < nFactors; ++k) { + factors.push_back(makeRandomVector(n, 1e-3, 1e-2, 200 + k)); + for (std::size_t i = 0; i < n; ++i) { + ref[i] += std::log(factors[k][i]); + if (k < nLogFactors) { + factors[k][i] = std::log(factors[k][i]); + } + } + } + // A zero and a negative factor. + factors[nLogFactors][0] = 0.0; + factors[nLogFactors][1] = -0.5; + + std::vector> vars(factors.begin(), factors.end()); + std::vector output(n); + std::array extraArgs{static_cast(nLogFactors)}; + RooBatchCompute::compute(makeConfig(1), RooBatchCompute::LogProdPdf, output, vars, extraArgs); + + EXPECT_EQ(output[0], -std::numeric_limits::infinity()); + EXPECT_TRUE(std::isnan(output[1])); + for (std::size_t i = 2; i < n; ++i) { + EXPECT_NEAR(output[i], ref[i], 1e-12 * std::abs(ref[i])) << "for event " << i; + } +} + +/// The logarithm of a product stays exact if a single factor is tiny, or if +/// the first factors multiply to a subnormal number or overflow and later +/// factors bring the product back into range. A NaN factor keeps its payload. +TEST_F(RooBatchComputeMT, ComputeLogProdPdfExtremeFactors) +{ + constexpr std::size_t nFactors = 207; + std::vector> factors(nFactors, std::vector(4, 1.0)); + for (std::size_t k = 0; k < 50; ++k) { + factors[k][0] = 0x1p-5; + } + factors[50][0] = 1e-247; + for (std::size_t k = 0; k < nFactors; ++k) { + factors[k][1] = k < 107 ? 1e-3 : 1e3; + factors[k][2] = k < 107 ? 1e3 : 1e-3; + } + factors[50][3] = RooNaNPacker::packFloatIntoNaN(3.f); + const std::array ref{50 * std::log(0x1p-5) + std::log(1e-247), 107 * std::log(1e-3) + 100 * std::log(1e3), + 107 * std::log(1e3) + 100 * std::log(1e-3)}; + + std::vector> vars(factors.begin(), factors.end()); + std::vector output(4); + std::array extraArgs{0.0}; + RooBatchCompute::compute(makeConfig(1), RooBatchCompute::LogProdPdf, output, vars, extraArgs); + + for (std::size_t i = 0; i < ref.size(); ++i) { + EXPECT_NEAR(output[i], ref[i], 1e-12 * std::abs(ref[i])) << "for event " << i; + } + EXPECT_TRUE(std::isnan(output[3])); +// Like in testNaNPacker, payloads don't survive arithmetic in this configuration. +#if !defined(_MSC_VER) || defined(_WIN64) || defined(NDEBUG) || defined(R__ENABLE_BROKEN_WIN_TESTS) + EXPECT_EQ(RooNaNPacker::unpackNaN(output[3]), 3.f); +#endif +} diff --git a/roofit/roofitcore/test/testRooEvaluator.cxx b/roofit/roofitcore/test/testRooEvaluator.cxx index 84fa3ff904209..9ffc50d300691 100644 --- a/roofit/roofitcore/test/testRooEvaluator.cxx +++ b/roofit/roofitcore/test/testRooEvaluator.cxx @@ -1,13 +1,17 @@ // Tests for RooFit::Evaluator // Author: Jonas Rembser, CERN 10/2026 +#include +#include #include #include #include #include +#include #include +#include /// Deleting nodes of the computation graph before the Evaluator must not /// crash: the Evaluator used to reset the data tokens of its nodes in the @@ -41,3 +45,28 @@ TEST(RooEvaluator, NodesDeletedBeforeEvaluator) RooFit::Evaluator evaluator3{*gauss2}; EXPECT_FLOAT_EQ(evaluator3.run()[0], gauss2->getVal()); } + +/// A node outputs its logarithm only if its single client can use it, and if +/// its values are not given as input. +TEST(RooEvaluator, LogEvaluation) +{ + RooRealVar x{"x", "x", 40.0, -50.0, 50.0}; + RooRealVar mu{"mu", "mu", 0.0, -1.0, 1.0}; + RooRealVar sigma{"sigma", "sigma", 1.0, 0.1, 10.0}; + RooGaussian gauss{"gauss", "gauss", x, mu, sigma}; + RooConstraintSum constraint{"constraint", "constraint", gauss, {}}; + + // The Gaussian underflows, but not its logarithm. + EXPECT_EQ(gauss.getVal(), 0.0); + EXPECT_DOUBLE_EQ(RooFit::Evaluator{constraint}.run()[0], 800.0); + + // The sum is a second client, which needs the value of the Gaussian. + x.setVal(1.0); + RooAddition sum{"sum", "sum", {constraint, gauss}}; + EXPECT_DOUBLE_EQ(RooFit::Evaluator{sum}.run()[0], 0.5 + std::exp(-0.5)); + + RooFit::Evaluator evaluator{constraint}; + std::vector input{0.25}; + evaluator.setInput("gauss", input, false); + EXPECT_DOUBLE_EQ(evaluator.run()[0], -std::log(0.25)); +} diff --git a/roofit/roofitcore/test/testRooProdPdf.cxx b/roofit/roofitcore/test/testRooProdPdf.cxx index 6bfae26d8dbc4..93b255b2d6fe6 100644 --- a/roofit/roofitcore/test/testRooProdPdf.cxx +++ b/roofit/roofitcore/test/testRooProdPdf.cxx @@ -168,6 +168,92 @@ TEST(RooProdPdf, NonFactorizingProductLikelihood) } } +/// The joint probability of a product of many pdfs, like an HS3 array +/// distribution, underflows, but the likelihood should still be finite. Covers +/// factors that can evaluate their logarithm directly (Gaussian), factors that +/// can't (Exponential), and a mix of both. Covers GitHub issue #24202. +TEST(RooProdPdf, ManyFactorsLikelihood) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + constexpr int nFactors = 300; + + for (std::string kind : {"Gaussian", "Exponential", "Mixed"}) { + RooWorkspace ws; + RooArgList factors; + RooArgSet obs; + for (int i = 0; i < nFactors; ++i) { + const std::string idx = std::to_string(i); + if (kind == "Gaussian" || (kind == "Mixed" && i % 2 == 0)) { + ws.factory("Gaussian::f" + idx + "(x" + idx + "[0, -10, 10], theta[0, -2, 2], 1.0)"); + } else { + ws.factory("Exponential::f" + idx + "(x" + idx + "[1, 0, 10], c[-1, -2, -0.5])"); + } + factors.add(*ws.pdf("f" + idx)); + obs.add(*ws.var("x" + idx)); + } + RooProdPdf prod{"prod", "prod", factors}; + + RooRealVar w{"w", "w", 1.0}; + RooArgSet vars{obs}; + vars.add(w); + RooDataSet data{"data", "data", vars, RooFit::WeightVar(w)}; + for (double val : {3.0, 4.0}) { + for (auto *x : static_range_cast(obs)) { + x->setVal(val); + } + data.add(obs, val); + } + + std::unique_ptr nll{prod.createNLL(data, RooFit::EvalBackend::Cpu())}; + + double ref = 0.0; + for (int i = 0; i < data.numEntries(); ++i) { + obs.assign(*data.get(i)); + for (auto *factor : static_range_cast(factors)) { + ref -= data.weight() * factor->getLogVal(&obs); + } + } + EXPECT_NEAR(nll->getVal(), ref, 1e-10 * std::abs(ref)) << kind; + } +} + +/// Like ManyFactorsLikelihood, but with the product as a constraint term. +TEST(RooProdPdf, ManyFactorsConstraint) +{ + RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + + constexpr int nFactors = 300; + + RooWorkspace ws; + ws.factory("Gaussian::model(x[0, -10, 10], theta[0.5, -2, 2], 1.0)"); + RooArgList constraints; + RooArgSet globs; + for (int i = 0; i < nFactors; ++i) { + const std::string idx = std::to_string(i); + ws.factory("Gaussian::c" + idx + "(g" + idx + "[3.5, -10, 10], theta, 1.0)"); + constraints.add(*ws.pdf("c" + idx)); + globs.add(*ws.var("g" + idx)); + } + RooProdPdf constraint{"constraint", "constraint", constraints}; + + RooRealVar &x = *ws.var("x"); + RooDataSet data{"data", "data", x}; + data.add(x); + + using namespace RooFit; + RooAbsPdf &model = *ws.pdf("model"); + std::unique_ptr nll{model.createNLL(data, EvalBackend::Cpu())}; + std::unique_ptr nllConstr{ + model.createNLL(data, EvalBackend::Cpu(), ExternalConstraints(constraint), GlobalObservables(globs))}; + + double ref = nll->getVal(); + for (auto *c : static_range_cast(constraints)) { + ref -= c->getLogVal(&globs); + } + EXPECT_NEAR(nllConstr->getVal(), ref, 1e-10 * std::abs(ref)); +} + /// This test covers a potential problem with the custom normalization ranges /// of PDFs: the value of a RooProdPdf should not depend on the normalization /// ranges of its clients. TODO: this is still a problem! It needs to be fixed, From 0f872bf1be7ae8581e70f991384df36a62631702 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Wed, 18 Nov 2026 17:14:54 +0000 Subject: [PATCH 3/6] [RF] Remove the comment annotations from the generated code The codegen backend wrapped the code of every node in "// Begin -- ..." and "// End -- ..." comments with the printed node. For big fits, the node names are long and obfuscated, so the comments made the code harder to read, not easier. Printing a node also evaluates it in scalar mode. Creating a likelihood could therefore report evaluation errors, e.g. for a product that underflows. The comments also ended the line of every piece of code. Now, the CodegenContext does that itself, such that a loop pragma can't end up behind other code on the same line. Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit 9a5ef56ea9968a5c346633cc9d57581f6101f2ae) --- roofit/roofitcore/inc/RooFit/CodegenContext.h | 13 ------ .../roofitcore/src/RooFit/CodegenContext.cxx | 44 ++++--------------- 2 files changed, 9 insertions(+), 48 deletions(-) diff --git a/roofit/roofitcore/inc/RooFit/CodegenContext.h b/roofit/roofitcore/inc/RooFit/CodegenContext.h index 1ef74da53de5c..d80e36636a855 100644 --- a/roofit/roofitcore/inc/RooFit/CodegenContext.h +++ b/roofit/roofitcore/inc/RooFit/CodegenContext.h @@ -115,17 +115,6 @@ class CodegenContext { std::string buildFunction(RooAbsArg const &arg, std::unordered_set const &dependsOnData = {}); - struct ScopeRAII { - std::string _fn; - CodegenContext &_ctx; - RooAbsArg const *_arg; - - public: - ScopeRAII(RooAbsArg const *arg, CodegenContext &ctx); - ~ScopeRAII(); - }; - ScopeRAII OutputScopeRangeComment(RooAbsArg const *arg) { return {arg, *this}; } - private: void pushScope(); void popScope(); @@ -189,8 +178,6 @@ class CodegenContext { std::unordered_set _dependsOnData; /// @brief The code layered by lexical scopes used as a stack. std::vector _code; - /// @brief The indentation level for pretty-printing. - unsigned _indent = 0; /// @brief Index to get unique names for temporary variables. mutable int _tmpVarIdx = 0; /// @brief A map to keep track of list names as assigned by addResult. diff --git a/roofit/roofitcore/src/RooFit/CodegenContext.cxx b/roofit/roofitcore/src/RooFit/CodegenContext.cxx index b234c41b216ec..6adad3c78d0b1 100644 --- a/roofit/roofitcore/src/RooFit/CodegenContext.cxx +++ b/roofit/roofitcore/src/RooFit/CodegenContext.cxx @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -78,8 +77,6 @@ std::string const &CodegenContext::getResult(RooAbsArg const &arg) throw std::runtime_error("You requested the result of a vector observable outside a loop scope for it!"); } - auto RAII(OutputScopeRangeComment(&arg)); - // Now, recursively call translate into the current argument to load the correct result. codegen(const_cast(arg), *this); @@ -135,24 +132,20 @@ void CodegenContext::addToCodeBody(RooAbsArg const *klass, std::string const &in /// @param isScopeIndep The value determining if the input is scope dependent. void CodegenContext::addToCodeBody(std::string const &in, bool isScopeIndep /* = false */) { - TString indented = in; - indented = indented.Strip(TString::kBoth); // trim - - std::string indent_str; - for (unsigned i = 0; i < _indent; ++i) - indent_str += " "; - indented = indented.Prepend(indent_str); - - // FIXME: Multiline input. - // indent_str += "\n"; - // indented = indented.ReplaceAll("\n", indent_str); + TString trimmed = in; + trimmed = trimmed.Strip(TString::kBoth); + // Always end the line, so that the code can't affect what is added next, + // like a preprocessor directive that has to start a line. + if (!trimmed.EndsWith("\n")) { + trimmed += "\n"; + } // If we are in a loop and the value is scope independent, save it at the top of the loop. // else, just save it in the current scope. if (_code.size() > 2 && isScopeIndep) { - _code[_code.size() - 2] += indented; + _code[_code.size() - 2] += trimmed; } else { - _code.back() += indented; + _code.back() += trimmed; } } @@ -306,25 +299,6 @@ std::string CodegenContext::buildArg(std::span arr) return "xlArr + " + offset; } -CodegenContext::ScopeRAII::ScopeRAII(RooAbsArg const *arg, CodegenContext &ctx) : _ctx(ctx), _arg(arg) -{ - std::ostringstream os; - os.imbue(std::locale::classic()); // the generated code is C++, not locale-dependent text - Option_t *opts = nullptr; - arg->printStream(os, _arg->defaultPrintContents(opts), _arg->defaultPrintStyle(opts)); - _fn = os.str(); - const std::string info = "// Begin -- " + _fn; - _ctx._indent++; - _ctx.addToCodeBody(_arg, info); -} - -CodegenContext::ScopeRAII::~ScopeRAII() -{ - const std::string info = "// End -- " + _fn + "\n"; - _ctx.addToCodeBody(_arg, info); - _ctx._indent--; -} - void CodegenContext::pushScope() { _code.push_back(""); From 97dab0338dd4b81d0309f88adeaf81ff95e78879 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 17 Nov 2026 17:42:01 +0000 Subject: [PATCH 4/6] [RF] Use logarithms of products and pdfs in codegen likelihoods Like the cpu backend, the code generated for the unbinned likelihood and the constraint sum now uses the logarithms of the pdf values directly. For that, the CodegenContext gets getLogResult(), which dispatches to codegenLogImpl() overloads and falls back to std::log() of the value. Products sum the logarithms of their factors, and Gaussians avoid exp(). The now unused MathFuncs::constraintSum() and the unbinned branch of MathFuncs::nll() are removed. The sums are generated with one statement per term. The compiler and Clad recurse for every operator of an expression, so a single expression with several thousand terms would overflow the stack. Reproducing that takes minutes, so the test checks the number of operators per generated statement instead. The likelihood of a product with many factors is then finite with codegen too. The gradients also get 1.7x faster for products of non-Gaussian pdfs and up to 10x for Gaussians. Without the gradient, a product of three non-Gaussian pdfs gets 7% slower, because of the additional logarithms. Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit e16b6adce3b574a570e993ebebd4e45c4deda199) --- roofit/codegen/inc/RooFit/CodegenImpl.h | 5 ++ roofit/codegen/src/CodegenImpl.cxx | 44 ++++++++++- roofit/codegen/test/testCodegen.cxx | 45 ++++++++++- roofit/roofitcore/inc/RooFit/CodegenContext.h | 7 ++ .../roofitcore/inc/RooFit/Detail/MathFuncs.h | 38 +++------- .../roofitcore/src/RooFit/CodegenContext.cxx | 69 +++++++++++------ roofit/roofitcore/src/RooNLLVarNew.cxx | 2 +- roofit/roofitcore/test/testRooProdPdf.cxx | 76 ++++++++++++++++--- 8 files changed, 222 insertions(+), 64 deletions(-) diff --git a/roofit/codegen/inc/RooFit/CodegenImpl.h b/roofit/codegen/inc/RooFit/CodegenImpl.h index 9e02b392cf17a..7a18af866c26a 100644 --- a/roofit/codegen/inc/RooFit/CodegenImpl.h +++ b/roofit/codegen/inc/RooFit/CodegenImpl.h @@ -132,6 +132,11 @@ void codegenImpl(RooStats::HistFactory::FlexibleInterpVar &arg, CodegenContext & void codegenImpl(RooUniform &arg, CodegenContext &ctx); void codegenImpl(RooWrapperPdf &arg, CodegenContext &ctx); +std::string codegenLogImpl(RooAbsArg &arg, CodegenContext &ctx); +std::string codegenLogImpl(RooFit::Detail::RooFixedProdPdf &arg, CodegenContext &ctx); +std::string codegenLogImpl(RooFit::Detail::RooNormalizedPdf &arg, CodegenContext &ctx); +std::string codegenLogImpl(RooGaussian &arg, CodegenContext &ctx); + std::string codegenIntegralImpl(RooAbsReal &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooBernstein &arg, int code, const char *rangeName, CodegenContext &ctx); std::string codegenIntegralImpl(RooBifurGauss &arg, int code, const char *rangeName, CodegenContext &ctx); diff --git a/roofit/codegen/src/CodegenImpl.cxx b/roofit/codegen/src/CodegenImpl.cxx index 91fd24b7ea6ef..eb10c50e85efb 100644 --- a/roofit/codegen/src/CodegenImpl.cxx +++ b/roofit/codegen/src/CodegenImpl.cxx @@ -73,6 +73,20 @@ namespace RooFit::Experimental { namespace { +/// Code for the sum of the logarithms of the given servers of `in`. Each term +/// is added in a separate statement, because the compiler and Clad recurse for +/// every operator of an expression, which overflows the stack for sums with +/// several thousand terms. +std::string sumOfLogs(RooAbsArg const &in, RooAbsCollection const &args, CodegenContext &ctx) +{ + const std::string sum = ctx.getTmpVarName(); + ctx.addToCodeBody(&in, "double " + sum + " = 0.0;\n"); + for (RooAbsArg *arg : args) { + ctx.addToCodeBody(&in, sum + " += " + ctx.getLogResult(*arg) + ";\n"); + } + return sum; +} + // Return a stringy-field version of the value, formatted to maximum precision. std::string doubleToString(double val) { @@ -428,7 +442,7 @@ void codegenImpl(RooConstVar &arg, CodegenContext &ctx) void codegenImpl(RooConstraintSum &arg, CodegenContext &ctx) { - ctx.addResult(&arg, ctx.buildCall(mathFunc("constraintSum"), arg.list(), arg.list().size())); + ctx.addResult(&arg, "-" + sumOfLogs(arg, arg.list(), ctx)); } // Generate RooFit codegen wrappers for RooFunctorBinding and similar objects, @@ -692,7 +706,10 @@ void codegenImpl(RooFit::Detail::RooNLLVarNew &arg, CodegenContext &ctx) // brackets of the loop is written at the end of the scopes lifetime. { auto scope = ctx.beginLoop(&arg); - std::string term = ctx.buildCall(mathFunc("nll"), arg.func(), arg.weightVar(), arg.binnedL(), 0); + // The unbinned likelihood takes the logarithm of the pdf directly, so + // that e.g. products of many factors don't underflow. + std::string term = arg.binnedL() ? ctx.buildCall(mathFunc("nll"), arg.func(), arg.weightVar(), 0) + : "-(" + ctx.getResult(arg.weightVar()) + ") * " + ctx.getLogResult(arg.func()); ctx.addToCodeBody(&arg, resName + " += " + term + ";"); } if (arg.expectedEvents()) { @@ -707,6 +724,29 @@ void codegenImpl(RooFit::Detail::RooNormalizedPdf &arg, CodegenContext &ctx) ctx.addResult(&arg, ctx.getResult(arg.pdf()) + "/" + ctx.getResult(arg.normIntegral())); } +std::string codegenLogImpl(RooAbsArg &arg, CodegenContext &ctx) +{ + return "std::log(" + ctx.getResult(arg) + ")"; +} + +std::string codegenLogImpl(RooFit::Detail::RooFixedProdPdf &arg, CodegenContext &ctx) +{ + if (arg.isRearranged()) { + return codegenLogImpl(static_cast(arg), ctx); + } + return sumOfLogs(arg, *arg.partList(), ctx); +} + +std::string codegenLogImpl(RooFit::Detail::RooNormalizedPdf &arg, CodegenContext &ctx) +{ + return ctx.getLogResult(arg.pdf()) + " - " + ctx.getLogResult(arg.normIntegral()); +} + +std::string codegenLogImpl(RooGaussian &arg, CodegenContext &ctx) +{ + return ctx.buildCall(mathFunc("logGaussian"), arg.getX(), arg.getMean(), arg.getSigma()); +} + void codegenImpl(RooParamHistFunc &arg, CodegenContext &ctx) { std::string const &idx = arg.dataHist().calculateTreeIndexForCodeSquash(ctx, arg.xList()); diff --git a/roofit/codegen/test/testCodegen.cxx b/roofit/codegen/test/testCodegen.cxx index f1156eabc750f..63de988969974 100644 --- a/roofit/codegen/test/testCodegen.cxx +++ b/roofit/codegen/test/testCodegen.cxx @@ -4,9 +4,12 @@ #include #include +#include #include #include +#include #include +#include #include #include @@ -14,7 +17,9 @@ #include +#include #include +#include #include #include #include @@ -59,13 +64,16 @@ std::string normalized(std::string const &code) /// it. The signature is the one that CodegenContext::buildFunction() emits. using CodegenFunc = double (*)(double *params, double const *obs, double const *xlArr); -CodegenFunc compileFunction(RooAbsArg &arg, RooArgList const ¶ms) +CodegenFunc compileFunction(RooAbsArg &arg, RooArgList const ¶ms, std::string *code = nullptr) { RooFit::Experimental::CodegenContext ctx; for (std::size_t i = 0; i < params.size(); ++i) { ctx.addResult(¶ms[i], "params[" + std::to_string(i) + "]"); } const std::string funcName = ctx.buildFunction(arg); + if (code) { + *code = ctx.collectedCode(); + } gInterpreter->Declare("#include \n"); if (!gInterpreter->Declare(ctx.collectedCode().c_str())) { ADD_FAILURE() << "generated code does not compile:\n" << ctx.collectedCode(); @@ -157,3 +165,38 @@ TEST(RooFitCodegen, LognormalParametrizations) << "useStandardParametrization = " << useStandardParametrization; } } + +// The compiler and Clad recurse for every operator of an expression. This +// overflows the stack if the sum of several thousand constraint terms is +// generated as a single expression, so each statement must stay small. +TEST(RooFitCodegen, ManyConstraints) +{ + constexpr int nConstraints = 1000; + RooRealVar theta{"theta", "theta", 0.3, -5.0, 5.0}; + RooArgSet globs; + RooArgSet constraints; + double ref = 0.0; + for (int i = 0; i < nConstraints; ++i) { + const std::string idx = std::to_string(i); + auto glob = std::make_unique(("glob" + idx).c_str(), "", 0.001 * i); + constraints.addOwned( + std::make_unique(("constraint" + idx).c_str(), "", *glob, theta, RooFit::RooConst(1.0))); + ref += 0.5 * (glob->getVal() - theta.getVal()) * (glob->getVal() - theta.getVal()); + globs.addOwned(std::move(glob)); + } + RooConstraintSum sum{"sum", "sum", constraints, {}}; + + std::string code; + CodegenFunc func = compileFunction(sum, theta, &code); + ASSERT_NE(func, nullptr); + double paramVals[] = {theta.getVal()}; + EXPECT_NEAR(func(paramVals, nullptr, nullptr), ref, 1e-12 * ref); + + std::stringstream statements{code}; + std::ptrdiff_t maxOperators = 0; + const auto isOperator = [](char c) { return c == '+' || c == '-'; }; + for (std::string statement; std::getline(statements, statement, ';');) { + maxOperators = std::max(maxOperators, std::count_if(statement.begin(), statement.end(), isOperator)); + } + EXPECT_LT(maxOperators, 10); +} diff --git a/roofit/roofitcore/inc/RooFit/CodegenContext.h b/roofit/roofitcore/inc/RooFit/CodegenContext.h index d80e36636a855..46e131ba71668 100644 --- a/roofit/roofitcore/inc/RooFit/CodegenContext.h +++ b/roofit/roofitcore/inc/RooFit/CodegenContext.h @@ -57,6 +57,11 @@ class CodegenContext { return getResult(key.arg()); } + /// Like getResult(), but for the logarithm of the value. Classes can + /// implement codegenLogImpl() to compute it directly, e.g. to avoid that + /// products of many factors underflow. + std::string const &getLogResult(RooAbsArg const &arg); + void addToGlobalScope(std::string const &str); void addVecObs(const char *key, int idx); int observableIndexOf(const RooAbsArg &arg) const; @@ -124,6 +129,7 @@ class CodegenContext { void endLoop(LoopScope const &scope); void addResult(TNamed const *key, std::string const &value); + std::string saveResult(RooAbsArg const *in, std::string const &value); template {}, bool>::type = true> std::string buildArg(T x) @@ -172,6 +178,7 @@ class CodegenContext { /// @brief Map of node names to their result strings. std::unordered_map _nodeNames; + std::unordered_map _logNodeNames; /// @brief A map to keep track of the observable indices if they are non scalar. std::unordered_map _vecObsIndices; /// @brief Indicate whether a node depends on the dataset. diff --git a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h index 7afba26486d4f..7cc6ab7086791 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h +++ b/roofit/roofitcore/inc/RooFit/Detail/MathFuncs.h @@ -183,19 +183,6 @@ double multipdf(int idx, DoubleArray pdfs) return pdfs[idx]; } -template -double constraintSum(DoubleArray comp, unsigned int compSize) -{ - double sum = 0; -#if defined(__CLING__) && defined(R__HAS_CLAD) -#pragma clad checkpoint loop -#endif - for (unsigned int i = 0; i < compSize; i++) { - sum -= std::log(comp[i]); - } - return sum; -} - inline unsigned int uniformBinNumber(double low, double high, double val, unsigned int numBins, double coef) { double binWidth = (high - low) / numBins; @@ -453,22 +440,19 @@ inline double chi2Asymmetric(double mu, double weight, double errLo, double errH return diff * diff / sigma2; } -inline double nll(double pdf, double weight, int binnedL, int doBinOffset) +/// The negative log-likelihood of one bin in a binned fit. +inline double nll(double pdf, double weight, int doBinOffset) { - if (binnedL) { - // Special handling of this case since std::log(Poisson(0,0)=0 but can't be - // calculated with usual log-formula since std::log(mu)=0. No update of result - // is required since term=0. - if (std::abs(pdf) < 1e-10 && std::abs(weight) < 1e-10) { - return 0.0; - } - if (doBinOffset) { - return pdf - weight - weight * (std::log(pdf) - std::log(weight)); - } - return pdf - weight * std::log(pdf) + TMath::LnGamma(weight + 1); - } else { - return -weight * std::log(pdf); + // Special handling of this case since std::log(Poisson(0,0)=0 but can't be + // calculated with usual log-formula since std::log(mu)=0. No update of result + // is required since term=0. + if (std::abs(pdf) < 1e-10 && std::abs(weight) < 1e-10) { + return 0.0; + } + if (doBinOffset) { + return pdf - weight - weight * (std::log(pdf) - std::log(weight)); } + return pdf - weight * std::log(pdf) + TMath::LnGamma(weight + 1); } template diff --git a/roofit/roofitcore/src/RooFit/CodegenContext.cxx b/roofit/roofitcore/src/RooFit/CodegenContext.cxx index 6adad3c78d0b1..a43768d5c123d 100644 --- a/roofit/roofitcore/src/RooFit/CodegenContext.cxx +++ b/roofit/roofitcore/src/RooFit/CodegenContext.cxx @@ -232,6 +232,13 @@ std::string CodegenContext::getTmpVarName() const /// @param in The node on which the valueToSave depends on/belongs to. /// @param valueToSave The actual string value to save as a temporary. void CodegenContext::addResult(RooAbsArg const *in, std::string const &valueToSave) +{ + addResult(in->namePtr(), saveResult(in, valueToSave)); +} + +/// @brief Save an expression that depends on the input node in a temporary if it is worth it. +/// @return The name of the temporary, or the expression itself. +std::string CodegenContext::saveResult(RooAbsArg const *in, std::string const &valueToSave) { // std::string savedName = RooFit::Detail::makeValidVarName(in->GetName()); std::string savedName = getTmpVarName(); @@ -254,7 +261,7 @@ void CodegenContext::addResult(RooAbsArg const *in, std::string const &valueToSa savedName = valueToSave; } - addResult(in->namePtr(), savedName); + return savedName; } /// @brief Function to save a RooListProxy as an array in the squashed code. @@ -399,36 +406,52 @@ struct Caller_FUNC_NAME { gInterpreter->Declare(dispatcherCode.c_str()); } -void codegen(RooAbsArg &arg, CodegenContext &ctx) -{ - static bool codeDeclared = false; - if (!codeDeclared) { - declareDispatcherCode("codegenImpl"); - codeDeclared = true; - } +namespace { - using Func = void (*)(RooAbsArg &, CodegenContext &); +/// Get the overload of the function `funcName` for the class of the given +/// argument, as resolved by the interpreter. +template +Func getDispatchedFunc(std::string const &funcName, RooAbsArg const &arg) +{ + // Cache the overload resolutions + static std::unordered_map> dispatchMaps; - Func func; + auto [entry, isNew] = dispatchMaps.try_emplace(funcName); + if (isNew) { + declareDispatcherCode(funcName); + } + auto &dispatchMap = entry->second; TClass *tclass = arg.IsA(); - - // Cache the overload resolutions - static std::unordered_map dispatchMap; - auto found = dispatchMap.find(tclass); - if (found != dispatchMap.end()) { - func = found->second; - } else { - // Can probably done with CppInterop in the future to avoid string manipulation. - std::stringstream cmd; - cmd << "&RooFit::Experimental::Caller_codegenImpl<" << tclass->GetName() << ">::call;"; - func = reinterpret_cast(gInterpreter->ProcessLine(cmd.str().c_str())); - dispatchMap[tclass] = func; + return found->second; } - return func(arg, ctx); + // Can probably done with CppInterop in the future to avoid string manipulation. + std::stringstream cmd; + cmd << "&RooFit::Experimental::Caller_" << funcName << "<" << tclass->GetName() << ">::call;"; + auto func = reinterpret_cast(gInterpreter->ProcessLine(cmd.str().c_str())); + dispatchMap[tclass] = func; + return func; +} + +} // namespace + +void codegen(RooAbsArg &arg, CodegenContext &ctx) +{ + getDispatchedFunc("codegenImpl", arg)(arg, ctx); +} + +std::string const &CodegenContext::getLogResult(RooAbsArg const &arg) +{ + auto found = _logNodeNames.find(arg.namePtr()); + if (found != _logNodeNames.end()) + return found->second; + + using Func = std::string (*)(RooAbsArg &, CodegenContext &); + const std::string logValue = getDispatchedFunc("codegenLogImpl", arg)(const_cast(arg), *this); + return _logNodeNames[arg.namePtr()] = saveResult(&arg, logValue); } } // namespace Experimental diff --git a/roofit/roofitcore/src/RooNLLVarNew.cxx b/roofit/roofitcore/src/RooNLLVarNew.cxx index b20acbfc1194b..2247ab30e0407 100644 --- a/roofit/roofitcore/src/RooNLLVarNew.cxx +++ b/roofit/roofitcore/src/RooNLLVarNew.cxx @@ -289,7 +289,7 @@ void RooNLLVarNew::doEvalBinnedL(RooFit::EvalContext &ctx, std::span #include +#include #include #include #include +#include TEST(RooProdPdf, TestGetPartIntList) { @@ -168,6 +171,54 @@ TEST(RooProdPdf, NonFactorizingProductLikelihood) } } +namespace { + +/// Checks a likelihood value with the cpu and codegen backends, and, if Clad +/// is available, the codegen gradient against numeric derivatives. +void checkLikelihoodBackends(std::function(RooFit::EvalBackend)> const &createNLL, + RooAbsData const &data, double ref, std::string const &label) +{ + // The underflow of the product must not cause evaluation errors anywhere. + const auto prevErrorLoggingMode = RooAbsReal::evalErrorLoggingMode(); + RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); + RooAbsReal::clearEvalErrorLog(); + + std::unique_ptr nll = createNLL(RooFit::EvalBackend::Cpu()); + EXPECT_NEAR(nll->getVal(), ref, 1e-10 * std::abs(ref)) << label << ", cpu backend"; + + std::unique_ptr nllCodegen = createNLL(RooFit::EvalBackend::CodegenNoGrad()); + EXPECT_NEAR(nllCodegen->getVal(), ref, 1e-10 * std::abs(ref)) << label << ", codegen backend"; + +#ifdef ROOFIT_CLAD + std::unique_ptr nllGrad = createNLL(RooFit::EvalBackend::Codegen()); + // Like in the wrapped function, all variables that are not in the dataset + // are parameters, including the global observables. + RooArgSet params; + nllGrad->getParameters(data.get(), params); + std::vector gradient(params.size()); + nllGrad->gradient(gradient.data()); + for (std::size_t i = 0; i < params.size(); ++i) { + auto ¶m = static_cast(*params[i]); + const double val = param.getVal(); + constexpr double eps = 1e-6; + param.setVal(val + eps); + const double up = nll->getVal(); + param.setVal(val - eps); + const double down = nll->getVal(); + param.setVal(val); + const double numDerivative = (up - down) / (2 * eps); + EXPECT_NEAR(gradient[i], numDerivative, 1e-5 * std::max(std::abs(numDerivative), 1.0)) + << label << ", codegen gradient by " << param.GetName(); + } +#endif + + EXPECT_EQ(RooAbsReal::numEvalErrors(), 0) << label; + RooAbsReal::clearEvalErrorLog(); + RooAbsReal::setEvalErrorLoggingMode(prevErrorLoggingMode); +} + +} // namespace + /// The joint probability of a product of many pdfs, like an HS3 array /// distribution, underflows, but the likelihood should still be finite. Covers /// factors that can evaluate their logarithm directly (Gaussian), factors that @@ -176,7 +227,7 @@ TEST(RooProdPdf, ManyFactorsLikelihood) { RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); - constexpr int nFactors = 300; + constexpr int nFactors = 100; for (std::string kind : {"Gaussian", "Exponential", "Mixed"}) { RooWorkspace ws; @@ -198,15 +249,13 @@ TEST(RooProdPdf, ManyFactorsLikelihood) RooArgSet vars{obs}; vars.add(w); RooDataSet data{"data", "data", vars, RooFit::WeightVar(w)}; - for (double val : {3.0, 4.0}) { + for (double val : {8.0, 9.0}) { for (auto *x : static_range_cast(obs)) { x->setVal(val); } data.add(obs, val); } - std::unique_ptr nll{prod.createNLL(data, RooFit::EvalBackend::Cpu())}; - double ref = 0.0; for (int i = 0; i < data.numEntries(); ++i) { obs.assign(*data.get(i)); @@ -214,7 +263,11 @@ TEST(RooProdPdf, ManyFactorsLikelihood) ref -= data.weight() * factor->getLogVal(&obs); } } - EXPECT_NEAR(nll->getVal(), ref, 1e-10 * std::abs(ref)) << kind; + + auto createNLL = [&](RooFit::EvalBackend backend) { + return std::unique_ptr{prod.createNLL(data, backend)}; + }; + checkLikelihoodBackends(createNLL, data, ref, kind); } } @@ -223,7 +276,7 @@ TEST(RooProdPdf, ManyFactorsConstraint) { RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); - constexpr int nFactors = 300; + constexpr int nFactors = 100; RooWorkspace ws; ws.factory("Gaussian::model(x[0, -10, 10], theta[0.5, -2, 2], 1.0)"); @@ -231,7 +284,7 @@ TEST(RooProdPdf, ManyFactorsConstraint) RooArgSet globs; for (int i = 0; i < nFactors; ++i) { const std::string idx = std::to_string(i); - ws.factory("Gaussian::c" + idx + "(g" + idx + "[3.5, -10, 10], theta, 1.0)"); + ws.factory("Gaussian::c" + idx + "(g" + idx + "[8.5, -10, 10], theta, 1.0)"); constraints.add(*ws.pdf("c" + idx)); globs.add(*ws.var("g" + idx)); } @@ -244,14 +297,17 @@ TEST(RooProdPdf, ManyFactorsConstraint) using namespace RooFit; RooAbsPdf &model = *ws.pdf("model"); std::unique_ptr nll{model.createNLL(data, EvalBackend::Cpu())}; - std::unique_ptr nllConstr{ - model.createNLL(data, EvalBackend::Cpu(), ExternalConstraints(constraint), GlobalObservables(globs))}; double ref = nll->getVal(); for (auto *c : static_range_cast(constraints)) { ref -= c->getLogVal(&globs); } - EXPECT_NEAR(nllConstr->getVal(), ref, 1e-10 * std::abs(ref)); + + auto createNLL = [&](RooFit::EvalBackend backend) { + return std::unique_ptr{ + model.createNLL(data, backend, ExternalConstraints(constraint), GlobalObservables(globs))}; + }; + checkLikelihoodBackends(createNLL, data, ref, "constraint"); } /// This test covers a potential problem with the custom normalization ranges From f45e7b08a336d898f5ffc98f2b51a30e52ea7a46 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Tue, 17 Nov 2026 23:39:13 +0000 Subject: [PATCH 5/6] [RF] Evaluate logarithms of products and pdfs also on the GPU With the cuda backend, the likelihood of a product with many factors was still infinite, because only nodes evaluated on the CPU output their logarithms. The Evaluator now sets the mode in both evaluation contexts, independent of the device. RooNormalizedPdf computes its logarithm with a new LogNormalizedPdf compute function instead of a host loop, so that it also works on the GPU. It first processes all events in a loop without branches and then recomputes the ones with evaluation errors or zero pdf values, which makes fits of one Gaussian with 1M events 10% faster than with the host loop. The other compute functions used in log mode already supported CUDA. The tests of products with many factors now check the likelihood with the cpu, cuda and codegen_no_grad backends, and the codegen gradient by comparing a Minuit 2 fit with strategy 0 to a fit with numeric derivatives. The data is chosen such that the minimum is within the parameter limits and the product underflows there. Assisted-by: claude-code:claude-fable-5-2 (cherry picked from commit c7e67b3c01a76eb48a89dc0b4dde288f2b0565a1) --- roofit/batchcompute/res/RooBatchCompute.h | 15 +- roofit/batchcompute/src/ComputeFunctions.cxx | 113 ++++++- roofit/batchcompute/src/RooBatchCompute.cu | 17 +- .../inc/RooFit/Detail/RooNormalizedPdf.h | 12 +- roofit/roofitcore/src/RooFit/Evaluator.cxx | 15 +- roofit/roofitcore/src/RooNormalizedPdf.cxx | 78 +---- roofit/roofitcore/test/testNaNPacker.cxx | 10 +- .../roofitcore/test/testRooBatchCompute.cxx | 63 ++++ roofit/roofitcore/test/testRooProdPdf.cxx | 302 +++++++++++------- 9 files changed, 397 insertions(+), 228 deletions(-) diff --git a/roofit/batchcompute/res/RooBatchCompute.h b/roofit/batchcompute/res/RooBatchCompute.h index 8cc6bbeb4786e..b02377e8aa52e 100644 --- a/roofit/batchcompute/res/RooBatchCompute.h +++ b/roofit/batchcompute/res/RooBatchCompute.h @@ -105,6 +105,7 @@ enum Computer { LognormalStandard, NegativeLogarithms, NormalizedPdf, + LogNormalizedPdf, Novosibirsk, Poisson, Polynomial, @@ -182,13 +183,13 @@ class RooBatchComputeInterface { /// Compute the values for a batch of events. /// /// The extra args (the last parameter) are read-only inputs for all - /// computers except `NormalizedPdf`, which uses them as output parameters - /// for its evaluation error counters. In the CUDA implementation, these - /// outputs are read back from the device *asynchronously*: they only - /// arrive in the caller's span with the next synchronizeCudaStream() call - /// on the stream of the passed config. The memory backing the extra args - /// of a `NormalizedPdf` call must therefore stay valid until that - /// synchronization, so it must not live on the caller's stack. + /// computers except `NormalizedPdf` and `LogNormalizedPdf`, which use the + /// first three as output parameters for their evaluation error counters. + /// In the CUDA implementation, these outputs are read back from the device + /// *asynchronously*: they only arrive in the caller's span with the next + /// synchronizeCudaStream() call on the stream of the passed config. The + /// memory backing the extra args of such a call must therefore stay valid + /// until that synchronization, so it must not live on the caller's stack. virtual void compute(Config const &cfg, Computer, std::span output, VarSpan, ArgSpan) = 0; virtual double reduceSum(Config const &cfg, InputArr input, size_t n) = 0; diff --git a/roofit/batchcompute/src/ComputeFunctions.cxx b/roofit/batchcompute/src/ComputeFunctions.cxx index 1f313fa4c0748..6774a8b1bbcea 100644 --- a/roofit/batchcompute/src/ComputeFunctions.cxx +++ b/roofit/batchcompute/src/ComputeFunctions.cxx @@ -644,6 +644,27 @@ __rooglobal__ void computeLognormalStandard(Batches &batches) } } +/// Add the evaluation error counts of the normalization compute functions to +/// the first three extra args. +__roodevice__ inline void addEvalErrorCounts(Batches &batches, int nType0, int nType1, int nType2) +{ + // The counters live in memory that is shared between all threads in the + // CUDA case, so they need to be accumulated atomically there. On the CPU, + // every task works on its own copy of the extra args. +#ifdef __CUDACC__ + if (nType0 > 0) + atomicAdd(&batches.extra[0], double(nType0)); + if (nType1 > 0) + atomicAdd(&batches.extra[1], double(nType1)); + if (nType2 > 0) + atomicAdd(&batches.extra[2], double(nType2)); +#else + batches.extra[0] = batches.extra[0] + nType0; + batches.extra[1] = batches.extra[1] + nType1; + batches.extra[2] = batches.extra[2] + nType2; +#endif +} + __rooglobal__ void computeNormalizedPdf(Batches &batches) { auto rawVal = batches.args[0]; @@ -674,23 +695,80 @@ __rooglobal__ void computeNormalizedPdf(Batches &batches) batches.output[i] = out; } - // The counters live in memory that is shared between all threads in the - // CUDA case, so they need to be accumulated atomically there. Note that - // the CPU branch below is only safe because the CPU implementation runs - // single-threaded: with implicit multi-threading, the workers would share - // this memory as well and would also need atomic accumulation. -#ifdef __CUDACC__ - if (nEvalErrorsType0 > 0) - atomicAdd(&batches.extra[0], double(nEvalErrorsType0)); - if (nEvalErrorsType1 > 0) - atomicAdd(&batches.extra[1], double(nEvalErrorsType1)); - if (nEvalErrorsType2 > 0) - atomicAdd(&batches.extra[2], double(nEvalErrorsType2)); -#else - batches.extra[0] = batches.extra[0] + nEvalErrorsType0; - batches.extra[1] = batches.extra[1] + nEvalErrorsType1; - batches.extra[2] = batches.extra[2] + nEvalErrorsType2; -#endif + addEvalErrorCounts(batches, nEvalErrorsType0, nEvalErrorsType1, nEvalErrorsType2); +} + +/// Logarithm of the normalized pdf for one event, with the same evaluation +/// errors as computeNormalizedPdf(). +template +__roodevice__ inline double logNormalizedPdf(double num, double den, double logDen, int *nEvalErrors) +{ + const bool numIsZero = NumIsLog ? num == -INFINITY : num == 0.; + const bool numIsNegative = !NumIsLog && num < 0.; + if (den < 0. || (den == 0. && !numIsZero)) { + nEvalErrors[0]++; + return RooNaNPacker::packFloatIntoNaN(-den + (numIsNegative ? -num : 0.)); + } + if (numIsNegative) { + nEvalErrors[1]++; + return RooNaNPacker::packFloatIntoNaN(-num); + } + if (std::isnan(num)) { + nEvalErrors[2]++; + return num; + } + return numIsZero ? -INFINITY : (NumIsLog ? num : std::log(num)) - logDen; +} + +/// The input modes are template parameters, so that no logarithms are +/// evaluated speculatively. +template +__roodevice__ inline void computeLogNormalizedPdfImpl(Batches &batches) +{ + auto rawVal = batches.args[0]; + auto normVal = batches.args[1]; + const double scalarLogDen = ScalarDen ? std::log(normVal[0]) : 0.; + int nEvalErrors[3] = {0, 0, 0}; + + // Loop without branches for the regular events, which the compiler can + // vectorize if no logarithms are taken. The other events, with evaluation + // errors or zero pdf values, are recomputed afterwards. + auto isRegular = [&](std::size_t i) { + return normVal[i] > 0. && (NumIsLog ? rawVal[i] > -INFINITY : rawVal[i] > 0.); + }; + bool allRegular = true; + for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { + const double logNum = NumIsLog ? rawVal[i] : std::log(rawVal[i]); + batches.output[i] = logNum - (ScalarDen ? scalarLogDen : std::log(normVal[i])); + allRegular &= isRegular(i); + } + if (!allRegular) { + for (size_t i = BEGIN; i < batches.nEvents; i += STEP) { + if (!isRegular(i)) { + const double logDen = ScalarDen ? scalarLogDen : std::log(normVal[i]); + batches.output[i] = logNormalizedPdf(rawVal[i], normVal[i], logDen, nEvalErrors); + } + } + } + + addEvalErrorCounts(batches, nEvalErrors[0], nEvalErrors[1], nEvalErrors[2]); +} + +/// Like computeNormalizedPdf(), but outputs the logarithm. The pdf values are +/// given as logarithms if `extra[3]` is nonzero. +__rooglobal__ void computeLogNormalizedPdf(Batches &batches) +{ + const bool numIsLog = batches.extra[3] != 0.; + const bool scalarDen = !batches.args[1]._isVector; + if (numIsLog && scalarDen) { + computeLogNormalizedPdfImpl(batches); + } else if (numIsLog) { + computeLogNormalizedPdfImpl(batches); + } else if (scalarDen) { + computeLogNormalizedPdfImpl(batches); + } else { + computeLogNormalizedPdfImpl(batches); + } } /* TMath::ASinH(x) needs to be replaced with ln( x + sqrt(x^2+1)) @@ -1042,6 +1120,7 @@ std::vector getFunctions() computeLognormalStandard, computeNegativeLogarithms, computeNormalizedPdf, + computeLogNormalizedPdf, computeNovosibirsk, computePoisson, computePolynomial, diff --git a/roofit/batchcompute/src/RooBatchCompute.cu b/roofit/batchcompute/src/RooBatchCompute.cu index 4d76088c0e4b8..7d047cb178074 100644 --- a/roofit/batchcompute/src/RooBatchCompute.cu +++ b/roofit/batchcompute/src/RooBatchCompute.cu @@ -289,14 +289,15 @@ public: const int gridSize = getGridSize(nEvents); _computeFunctions[computer]<<>>(*batchesDevice); - // Only the NormalizedPdf computer mutates its extra args: it uses them - // as output parameters for the evaluation error counts. Instead of - // synchronizing the stream to read the counters back immediately, the - // readback is deferred to avoid stalling the pipeline: an asynchronous - // copy delivers them into a persistent pinned buffer, and the next - // synchronizeCudaStream() call forwards them to the caller's span. The - // caller's memory therefore has to stay valid until then. - if (computer == NormalizedPdf && !extraArgs.empty()) { + // Only the NormalizedPdf and LogNormalizedPdf computers mutate their + // extra args: they use them as output parameters for the evaluation + // error counts. Instead of synchronizing the stream to read the counters + // back immediately, the readback is deferred to avoid stalling the + // pipeline: an asynchronous copy delivers them into a persistent pinned + // buffer, and the next synchronizeCudaStream() call forwards them to the + // caller's span. The caller's memory therefore has to stay valid until + // then. + if ((computer == NormalizedPdf || computer == LogNormalizedPdf) && !extraArgs.empty()) { const std::size_t nBytes = extraArgs.size() * sizeof(double); StreamScratch::DeferredSlot &deferredSlot = streamScratch.acquireDeferred(nBytes); ERRCHECK(cudaMemcpyAsync(deferredSlot.host, extraArgsDevice, nBytes, cudaMemcpyDeviceToHost, stream)); diff --git a/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h b/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h index 99cbeee768a80..47495d728119f 100644 --- a/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h +++ b/roofit/roofitcore/inc/RooFit/Detail/RooNormalizedPdf.h @@ -108,11 +108,13 @@ class RooNormalizedPdf : public RooAbsPdf { RooRealProxy _normIntegral; RooArgSet _normSet; - /// Evaluation error counters, filled by the compute function. In CUDA - /// mode, they are read back from the GPU asynchronously and only arrive - /// after the evaluation of the computation graph, so they have to live in - /// a member and not on the stack of doEval(). Transient and not copied. - mutable std::array _evalErrorCounts{}; // _extraArgs{}; // log() roundtrips, e.g. for Gaussian constraints, and the /// underflow of products with many factors. Nodes that output their /// logarithms can request the same from their servers, e.g. the factors of a -/// product. Only implemented for nodes evaluated on the CPU. +/// product. void Evaluator::setupLogEvaluation() { // Visit clients before their servers, because whether a client can use a @@ -269,13 +269,16 @@ void Evaluator::setupLogEvaluation() for (auto iter = _nodes.rbegin(); iter != _nodes.rend(); ++iter) { NodeInfo &info = *iter; bool outputsLogarithm = false; - if (info.clientInfos.size() == 1 && !info.fromArrayInput && !info.computeInGPU && - info.absArg->canOutputLogarithm()) { + if (info.clientInfos.size() == 1 && !info.fromArrayInput && info.absArg->canOutputLogarithm()) { NodeInfo const &client = *info.clientInfos[0]; - outputsLogarithm = !client.computeInGPU && client.absArg->canUseLogarithmOf( - *info.absArg, _evalContextCPU._outputsLogarithm[client.iNode]); + outputsLogarithm = + client.absArg->canUseLogarithmOf(*info.absArg, _evalContextCPU._outputsLogarithm[client.iNode]); } + // The output of a node can be read on both devices. _evalContextCPU._outputsLogarithm[info.iNode] = outputsLogarithm; + if (_useGPU) { + _evalContextCUDA._outputsLogarithm[info.iNode] = outputsLogarithm; + } } } @@ -369,7 +372,7 @@ void Evaluator::updateOutputSizes() markGPUNodes(); } - // Which nodes are inputs or evaluated on the GPU might have changed. + // Which nodes are array inputs might have changed. setupLogEvaluation(); _needToUpdateOutputSizes = false; diff --git a/roofit/roofitcore/src/RooNormalizedPdf.cxx b/roofit/roofitcore/src/RooNormalizedPdf.cxx index 86cba72c1e199..dacba0aed94bd 100644 --- a/roofit/roofitcore/src/RooNormalizedPdf.cxx +++ b/roofit/roofitcore/src/RooNormalizedPdf.cxx @@ -14,11 +14,8 @@ #include "RooBatchCompute.h" #include "RooFitImplHelpers.h" -#include "RooNaNPacker.h" #include -#include -#include /** * \class RooNormalizedPdf @@ -27,74 +24,27 @@ * normalization set into a new self-normalized pdf. */ -namespace { - -/// Logarithm of the normalized pdf, with the same evaluation errors as the -/// NormalizedPdf compute function. The input modes are template parameters, -/// so that no logarithms are evaluated speculatively. -template -void evalLogNormalized(std::span output, std::span nums, std::span dens, - std::array &errorCounts) -{ - constexpr double negInf = -std::numeric_limits::infinity(); - const double scalarLogDen = ScalarDen ? std::log(dens[0]) : 0.; - for (std::size_t i = 0; i < output.size(); ++i) { - const double num = nums.size() > 1 ? nums[i] : nums[0]; - const double den = ScalarDen ? dens[0] : dens[i]; - const bool numIsZero = NumIsLog ? num == negInf : num == 0.; - const bool numIsNegative = !NumIsLog && num < 0.; - if (den < 0. || (den == 0. && !numIsZero)) { - output[i] = RooNaNPacker::packFloatIntoNaN(-den + (numIsNegative ? -num : 0.)); - ++errorCounts[0]; - } else if (numIsNegative) { - output[i] = RooNaNPacker::packFloatIntoNaN(-num); - ++errorCounts[1]; - } else if (std::isnan(num)) { - output[i] = num; - ++errorCounts[2]; - } else if (numIsZero) { - output[i] = negInf; - } else { - const double logNum = NumIsLog ? num : std::log(num); - const double logDen = ScalarDen ? scalarLogDen : std::log(den); - output[i] = logNum - logDen; - } - } -} - -} // namespace - namespace RooFit::Detail { void RooNormalizedPdf::doEval(RooFit::EvalContext &ctx) const { auto nums = ctx.at(_pdf); auto integralSpan = ctx.at(_normIntegral); - - // We use the extraArgs as output parameter to count evaluation errors. - _evalErrorCounts = {}; - - if (ctx.outputsLogarithm(this)) { - // The pdf might provide its logarithm already. - const bool numIsLog = ctx.outputsLogarithm(&*_pdf); - const bool scalarDen = integralSpan.size() == 1; - auto evalLog = numIsLog ? (scalarDen ? evalLogNormalized : evalLogNormalized) - : (scalarDen ? evalLogNormalized : evalLogNormalized); - evalLog(ctx.output(), nums, integralSpan, _evalErrorCounts); - // The logarithm is only requested in CPU evaluation. - logEvalErrorCounts(); - return; - } - auto config = ctx.config(this); - RooBatchCompute::compute(config, RooBatchCompute::NormalizedPdf, ctx.output(), {nums, integralSpan}, - _evalErrorCounts); + + // The first three extra args are output parameters to count evaluation + // errors. The last one tells the LogNormalizedPdf computer whether the pdf + // values are logarithms already. + _extraArgs = {0., 0., 0., static_cast(ctx.outputsLogarithm(&*_pdf))}; + const auto computer = + ctx.outputsLogarithm(this) ? RooBatchCompute::LogNormalizedPdf : RooBatchCompute::NormalizedPdf; + RooBatchCompute::compute(config, computer, ctx.output(), {nums, integralSpan}, _extraArgs); if (config.useCuda()) { // In CUDA mode, the counters are read back from the GPU without - // synchronizing the stream: they only arrive in _evalErrorCounts with - // the synchronization at the end of the evaluation of the full - // computation graph, so the logging has to be deferred until then. + // synchronizing the stream: they only arrive in _extraArgs with the + // synchronization at the end of the evaluation of the full computation + // graph, so the logging has to be deferred until then. ctx.deferAction([this] { logEvalErrorCounts(); }); } else { logEvalErrorCounts(); @@ -103,9 +53,9 @@ void RooNormalizedPdf::doEval(RooFit::EvalContext &ctx) const void RooNormalizedPdf::logEvalErrorCounts() const { - const std::size_t nEvalErrorsType0 = _evalErrorCounts[0]; - const std::size_t nEvalErrorsType1 = _evalErrorCounts[1]; - const std::size_t nEvalErrorsType2 = _evalErrorCounts[2]; + const std::size_t nEvalErrorsType0 = _extraArgs[0]; + const std::size_t nEvalErrorsType1 = _extraArgs[1]; + const std::size_t nEvalErrorsType2 = _extraArgs[2]; for (std::size_t i = 0; i < nEvalErrorsType0; ++i) { logEvalError("p.d.f normalization integral is zero or negative"); diff --git a/roofit/roofitcore/test/testNaNPacker.cxx b/roofit/roofitcore/test/testNaNPacker.cxx index 4e38933d07933..0fa543157d332 100644 --- a/roofit/roofitcore/test/testNaNPacker.cxx +++ b/roofit/roofitcore/test/testNaNPacker.cxx @@ -304,10 +304,11 @@ TEST(RooNaNPacker, CudaEvalErrorParity) /// A negative normalization integral is packed into the NaN as badness, also /// if the pdf is evaluated in logarithmic scale. -TEST(RooNaNPacker, NegativeNormIntegralBadness) +TEST_P(TestForDifferentBackends, NegativeNormIntegralBadness) { const auto prevErrorLoggingMode = RooAbsReal::evalErrorLoggingMode(); - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::Ignore); + RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); + RooAbsReal::clearEvalErrorLog(); RooWorkspace ws; ws.factory("Gaussian::gauss(x[0, -10, 10], mean[0, -5, 5], sigma[1, -5, 5])"); @@ -318,14 +319,17 @@ TEST(RooNaNPacker, NegativeNormIntegralBadness) data.add(x); } - std::unique_ptr nll{ws.pdf("gauss")->createNLL(data, RooFit::EvalBackend::Cpu())}; + std::unique_ptr nll{ws.pdf("gauss")->createNLL(data, _evalBackend)}; ws.var("sigma")->setVal(-1.0); const double val = nll->getVal(); // Each event contributes the negative integral sqrt(2 pi) * sigma. EXPECT_TRUE(RooNaNPacker::isNaNWithPayload(val)); EXPECT_NEAR(RooNaNPacker::unpackNaN(val), 3 * std::sqrt(2 * TMath::Pi()), 1e-5); + // Each event logs one error for the integral and one for the likelihood. + EXPECT_EQ(RooAbsReal::numEvalErrors(), 6); + RooAbsReal::clearEvalErrorLog(); RooAbsReal::setEvalErrorLoggingMode(prevErrorLoggingMode); } diff --git a/roofit/roofitcore/test/testRooBatchCompute.cxx b/roofit/roofitcore/test/testRooBatchCompute.cxx index 10ac869a14f96..2bf8b73672ef8 100644 --- a/roofit/roofitcore/test/testRooBatchCompute.cxx +++ b/roofit/roofitcore/test/testRooBatchCompute.cxx @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include namespace { @@ -251,3 +253,64 @@ TEST_F(RooBatchComputeMT, ComputeLogProdPdfExtremeFactors) EXPECT_EQ(RooNaNPacker::unpackNaN(output[3]), 3.f); #endif } + +/// The LogNormalizedPdf computer outputs the logarithm of the NormalizedPdf +/// output, with the same NaN payloads and evaluation error counts, for pdf +/// values given linearly or as logarithms, and for scalar integrals. +TEST_F(RooBatchComputeMT, ComputeLogNormalizedPdf) +{ + std::vector nums = makeRandomVector(nEvents, 0., 10., 107); + std::vector dens = makeRandomVector(nEvents, 0.5, 2., 108); + // Scalar inputs are broadcast to bufferSize-wide arrays. + std::vector scalarDens(RooBatchCompute::bufferSize, 1.5); + // All types of evaluation errors, and zero pdf values. + dens[10] = -1.0; + dens[11] = 0.0; + nums[12] = std::numeric_limits::quiet_NaN(); + nums[13] = 0.0; + nums[14] = 0.0; + dens[14] = 0.0; + std::vector logNums(nEvents); + std::transform(nums.begin(), nums.end(), logNums.begin(), [](double x) { return std::log(x); }); + // Negative pdf values can only be given linearly. + std::vector numsWithNegative = nums; + numsWithNegative[15] = -0.5; + + auto run = [](RooBatchCompute::Computer computer, std::vector const &x, std::vector const &den, + bool numIsLog, int nThreads) { + std::vector output(nEvents); + std::vector extraArgs{0.0, 0.0, 0.0, static_cast(numIsLog)}; + RooBatchCompute::compute(makeConfig(nThreads), computer, output, {x, den}, extraArgs); + return std::make_pair(output, extraArgs); + }; + + for (bool numIsLog : {false, true}) { + std::vector const &linearNums = numIsLog ? nums : numsWithNegative; + std::vector const &input = numIsLog ? logNums : numsWithNegative; + for (std::vector const *den : {&dens, &scalarDens}) { + auto [ref, refCounts] = run(RooBatchCompute::NormalizedPdf, linearNums, *den, false, 1); + EXPECT_GT(refCounts[0] + refCounts[1] + refCounts[2], 0.0); // the test must produce errors + for (int nThreads : {1, 4}) { + auto [out, counts] = run(RooBatchCompute::LogNormalizedPdf, input, *den, numIsLog, nThreads); + std::stringstream ctx; + ctx << "with numIsLog = " << numIsLog << ", scalar integral = " << (den == &scalarDens) << ", " << nThreads + << " threads"; + for (std::size_t k = 0; k < 3; ++k) { + EXPECT_EQ(counts[k], refCounts[k]) << "evaluation errors of type " << k << " " << ctx.str(); + } + EXPECT_EQ(counts[3], static_cast(numIsLog)) << ctx.str(); + for (std::size_t i = 0; i < nEvents; ++i) { + if (std::isnan(ref[i])) { + ASSERT_TRUE(std::isnan(out[i])) << "for event " << i << " " << ctx.str(); + ASSERT_EQ(RooNaNPacker::unpackNaN(out[i]), RooNaNPacker::unpackNaN(ref[i])) + << "for event " << i << " " << ctx.str(); + } else if (ref[i] == 0.0) { + ASSERT_EQ(out[i], -std::numeric_limits::infinity()) << "for event " << i << " " << ctx.str(); + } else { + ASSERT_NEAR(out[i], std::log(ref[i]), 1e-14) << "for event " << i << " " << ctx.str(); + } + } + } + } + } +} diff --git a/roofit/roofitcore/test/testRooProdPdf.cxx b/roofit/roofitcore/test/testRooProdPdf.cxx index b01416dd88e79..d31b612075adc 100644 --- a/roofit/roofitcore/test/testRooProdPdf.cxx +++ b/roofit/roofitcore/test/testRooProdPdf.cxx @@ -13,19 +13,19 @@ #include #include #include +#include #include #include #include "gtest_wrapper.h" -#include +#include #include -#include #include #include #include -#include +#include TEST(RooProdPdf, TestGetPartIntList) { @@ -173,143 +173,209 @@ TEST(RooProdPdf, NonFactorizingProductLikelihood) namespace { -/// Checks a likelihood value with the cpu and codegen backends, and, if Clad -/// is available, the codegen gradient against numeric derivatives. -void checkLikelihoodBackends(std::function(RooFit::EvalBackend)> const &createNLL, - RooAbsData const &data, double ref, std::string const &label) -{ - // The underflow of the product must not cause evaluation errors anywhere. - const auto prevErrorLoggingMode = RooAbsReal::evalErrorLoggingMode(); - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); - RooAbsReal::clearEvalErrorLog(); - - std::unique_ptr nll = createNLL(RooFit::EvalBackend::Cpu()); - EXPECT_NEAR(nll->getVal(), ref, 1e-10 * std::abs(ref)) << label << ", cpu backend"; - - std::unique_ptr nllCodegen = createNLL(RooFit::EvalBackend::CodegenNoGrad()); - EXPECT_NEAR(nllCodegen->getVal(), ref, 1e-10 * std::abs(ref)) << label << ", codegen backend"; - -#ifdef ROOFIT_CLAD - std::unique_ptr nllGrad = createNLL(RooFit::EvalBackend::Codegen()); - // Like in the wrapped function, all variables that are not in the dataset - // are parameters, including the global observables. - RooArgSet params; - nllGrad->getParameters(data.get(), params); - std::vector gradient(params.size()); - nllGrad->gradient(gradient.data()); - for (std::size_t i = 0; i < params.size(); ++i) { - auto ¶m = static_cast(*params[i]); - const double val = param.getVal(); - constexpr double eps = 1e-6; - param.setVal(val + eps); - const double up = nll->getVal(); - param.setVal(val - eps); - const double down = nll->getVal(); - param.setVal(val); - const double numDerivative = (up - down) / (2 * eps); - EXPECT_NEAR(gradient[i], numDerivative, 1e-5 * std::max(std::abs(numDerivative), 1.0)) - << label << ", codegen gradient by " << param.GetName(); +/// A product of many factors, like an HS3 array distribution, with data for +/// which the product underflows to zero for every event. Also at the minimum +/// of the likelihood, which is within the parameter limits. Covers factors +/// that can evaluate their logarithm directly ("Gaussian"), factors that can't +/// ("Exponential"), a mix of both ("Mixed"), and the product as a constraint +/// term ("Constraint"). Covers GitHub issue #24202. +class ManyFactorsModel { +public: + explicit ManyFactorsModel(std::string const &kind) : _kind{kind} + { + constexpr int nFactors = 100; + + if (_kind == "Constraint") { + _ws.factory("Gaussian::model(x[0, -10, 10], theta[-1, -2, 2], 1.0)"); + for (int i = 0; i < nFactors; ++i) { + const std::string idx = std::to_string(i); + const char *globVal = i % 2 == 0 ? "8" : "-7"; + _ws.factory("Gaussian::f" + idx + "(g" + idx + "[" + globVal + ", -10, 10], theta, 1.0)"); + _factors.add(*_ws.pdf("f" + idx)); + _obs.add(*_ws.var("g" + idx)); + } + _obs.setAttribAll("Constant", true); + RooRealVar &x = *_ws.var("x"); + _data = std::make_unique("data", "data", x); + _data->add(x); + } else { + for (int i = 0; i < nFactors; ++i) { + const std::string idx = std::to_string(i); + if (isGaussian(i)) { + _ws.factory("Gaussian::f" + idx + "(x" + idx + "[0, -10, 10], theta[-1, -2, 2], 1.0)"); + } else { + // The wide range makes the exponential pdf values small. + _ws.factory("Exponential::f" + idx + "(x" + idx + "[1, 0, 10000], c[-1e-4, -1e-2, -1e-5])"); + } + _factors.add(*_ws.pdf("f" + idx)); + _obs.add(*_ws.var("x" + idx)); + } + _ws.factory("w[1.0]"); + RooArgSet vars{_obs, *_ws.var("w")}; + _data = std::make_unique("data", "data", vars, RooFit::WeightVar("w")); + // Values of the Gaussian and exponential observables, and weight. + using Event = std::array; + for (auto [gaussVal, expVal, weight] : {Event{-8., 100., 1.5}, Event{9., 6000., 1.}}) { + for (int i = 0; i < nFactors; ++i) { + static_cast(*_obs[i]).setVal(isGaussian(i) ? gaussVal : expVal); + } + _data->add(_obs, weight); + } + } + _prod = std::make_unique("prod", "prod", _factors); } -#endif - EXPECT_EQ(RooAbsReal::numEvalErrors(), 0) << label; - RooAbsReal::clearEvalErrorLog(); - RooAbsReal::setEvalErrorLoggingMode(prevErrorLoggingMode); -} - -} // namespace - -/// The joint probability of a product of many pdfs, like an HS3 array -/// distribution, underflows, but the likelihood should still be finite. Covers -/// factors that can evaluate their logarithm directly (Gaussian), factors that -/// can't (Exponential), and a mix of both. Covers GitHub issue #24202. -TEST(RooProdPdf, ManyFactorsLikelihood) -{ - RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + std::unique_ptr createNLL(RooFit::EvalBackend backend) const + { + using namespace RooFit; + if (_kind == "Constraint") { + return std::unique_ptr{ + _ws.pdf("model")->createNLL(*_data, backend, ExternalConstraints(*_prod), GlobalObservables(_obs))}; + } + return std::unique_ptr{_prod->createNLL(*_data, backend)}; + } - constexpr int nFactors = 100; - - for (std::string kind : {"Gaussian", "Exponential", "Mixed"}) { - RooWorkspace ws; - RooArgList factors; - RooArgSet obs; - for (int i = 0; i < nFactors; ++i) { - const std::string idx = std::to_string(i); - if (kind == "Gaussian" || (kind == "Mixed" && i % 2 == 0)) { - ws.factory("Gaussian::f" + idx + "(x" + idx + "[0, -10, 10], theta[0, -2, 2], 1.0)"); - } else { - ws.factory("Exponential::f" + idx + "(x" + idx + "[1, 0, 10], c[-1, -2, -0.5])"); + /// The likelihood, computed from the logarithms of the factors. + double referenceNLL() + { + double nll = 0.0; + if (_kind == "Constraint") { + nll = createNLLWithoutConstraint()->getVal(); + for (auto *factor : static_range_cast(_factors)) { + nll -= factor->getLogVal(&_obs); } - factors.add(*ws.pdf("f" + idx)); - obs.add(*ws.var("x" + idx)); + return nll; } - RooProdPdf prod{"prod", "prod", factors}; - - RooRealVar w{"w", "w", 1.0}; - RooArgSet vars{obs}; - vars.add(w); - RooDataSet data{"data", "data", vars, RooFit::WeightVar(w)}; - for (double val : {8.0, 9.0}) { - for (auto *x : static_range_cast(obs)) { - x->setVal(val); + for (int i = 0; i < _data->numEntries(); ++i) { + _obs.assign(*_data->get(i)); + for (auto *factor : static_range_cast(_factors)) { + nll -= _data->weight() * factor->getLogVal(&_obs); } - data.add(obs, val); } + return nll; + } - double ref = 0.0; - for (int i = 0; i < data.numEntries(); ++i) { - obs.assign(*data.get(i)); - for (auto *factor : static_range_cast(factors)) { - ref -= data.weight() * factor->getLogVal(&obs); - } + /// The tests are only meaningful if the product underflows. + void expectProductUnderflows() + { + if (_kind == "Constraint") { + EXPECT_EQ(_prod->getVal(_obs), 0.0) << "constraint"; + return; + } + for (int i = 0; i < _data->numEntries(); ++i) { + _obs.assign(*_data->get(i)); + EXPECT_EQ(_prod->getVal(_obs), 0.0) << "event " << i; } - - auto createNLL = [&](RooFit::EvalBackend backend) { - return std::unique_ptr{prod.createNLL(data, backend)}; - }; - checkLikelihoodBackends(createNLL, data, ref, kind); } -} -/// Like ManyFactorsLikelihood, but with the product as a constraint term. -TEST(RooProdPdf, ManyFactorsConstraint) -{ - RooHelpers::LocalChangeMsgLevel changeMsgLvl(RooFit::WARNING); + RooArgSet allVars() const { return _ws.allVars(); } - constexpr int nFactors = 100; +private: + bool isGaussian(int i) const { return _kind == "Gaussian" || (_kind == "Mixed" && i % 2 == 0); } - RooWorkspace ws; - ws.factory("Gaussian::model(x[0, -10, 10], theta[0.5, -2, 2], 1.0)"); - RooArgList constraints; - RooArgSet globs; - for (int i = 0; i < nFactors; ++i) { - const std::string idx = std::to_string(i); - ws.factory("Gaussian::c" + idx + "(g" + idx + "[8.5, -10, 10], theta, 1.0)"); - constraints.add(*ws.pdf("c" + idx)); - globs.add(*ws.var("g" + idx)); + std::unique_ptr createNLLWithoutConstraint() const + { + return std::unique_ptr{_ws.pdf("model")->createNLL(*_data, RooFit::EvalBackend::Cpu())}; } - RooProdPdf constraint{"constraint", "constraint", constraints}; - RooRealVar &x = *ws.var("x"); - RooDataSet data{"data", "data", x}; - data.add(x); + std::string _kind; + RooWorkspace _ws; + RooArgList _factors; + RooArgSet _obs; // the observables of the product, global observables for "Constraint" + std::unique_ptr _data; + std::unique_ptr _prod; +}; - using namespace RooFit; - RooAbsPdf &model = *ws.pdf("model"); - std::unique_ptr nll{model.createNLL(data, EvalBackend::Cpu())}; +const auto manyFactorsKinds = testing::Values("Gaussian", "Exponential", "Mixed", "Constraint"); + +} // namespace + +class ManyFactorsTest : public testing::TestWithParam> { +protected: + void SetUp() override + { + _changeMsgLvl = std::make_unique(RooFit::WARNING); + _prevErrorLoggingMode = RooAbsReal::evalErrorLoggingMode(); + RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); + RooAbsReal::clearEvalErrorLog(); + } - double ref = nll->getVal(); - for (auto *c : static_range_cast(constraints)) { - ref -= c->getLogVal(&globs); + void TearDown() override + { + RooAbsReal::clearEvalErrorLog(); + RooAbsReal::setEvalErrorLoggingMode(_prevErrorLoggingMode); + _changeMsgLvl.reset(); } - auto createNLL = [&](RooFit::EvalBackend backend) { - return std::unique_ptr{ - model.createNLL(data, backend, ExternalConstraints(constraint), GlobalObservables(globs))}; +private: + std::unique_ptr _changeMsgLvl; + RooAbsReal::ErrorLoggingMode _prevErrorLoggingMode = RooAbsReal::PrintErrors; +}; + +/// The likelihood of a product that underflows must still be finite, without +/// evaluation errors. +TEST_P(ManyFactorsTest, Likelihood) +{ + ManyFactorsModel model{std::get<0>(GetParam())}; + model.expectProductUnderflows(); + const double ref = model.referenceNLL(); + RooAbsReal::clearEvalErrorLog(); + + std::unique_ptr nll = model.createNLL(std::get<1>(GetParam())); + EXPECT_NEAR(nll->getVal(), ref, 1e-10 * std::abs(ref)); + EXPECT_EQ(RooAbsReal::numEvalErrors(), 0); +} + +INSTANTIATE_TEST_SUITE_P(RooProdPdf, ManyFactorsTest, + testing::Combine(manyFactorsKinds, + testing::Values(ROOFIT_EVAL_BACKEND_CUDA RooFit::EvalBackend::Cpu(), + RooFit::EvalBackend::CodegenNoGrad())), + [](testing::TestParamInfo const ¶mInfo) { + return std::get<0>(paramInfo.param) + "_EvalBackend" + std::get<1>(paramInfo.param).name(); + }); + +#ifdef ROOFIT_CLAD +using ManyFactorsGradientTest = ManyFactorsTest; + +/// The gradient of the codegen backend must be correct where the product +/// underflows: a fit that uses it must agree with a fit that uses numeric +/// derivatives. +TEST_P(ManyFactorsGradientTest, CodegenFit) +{ + ManyFactorsModel model{std::get<0>(GetParam())}; + std::unique_ptr nllRef = model.createNLL(RooFit::EvalBackend::Cpu()); + std::unique_ptr nll = model.createNLL(std::get<1>(GetParam())); + ASSERT_TRUE(nll->hasGradient()); + + RooArgSet vars = model.allVars(); + RooArgSet initialValues; + vars.snapshot(initialValues); + + auto fit = [&](RooAbsReal &absReal) { + vars.assign(initialValues); + RooMinimizer minimizer{absReal}; + minimizer.setPrintLevel(-1); + minimizer.setStrategy(0); + minimizer.minimize("Minuit2"); + return std::unique_ptr{minimizer.save()}; }; - checkLikelihoodBackends(createNLL, data, ref, "constraint"); + std::unique_ptr resultRef = fit(*nllRef); + std::unique_ptr result = fit(*nll); + + EXPECT_EQ(resultRef->status(), 0); + EXPECT_EQ(result->status(), 0); + EXPECT_TRUE(result->isIdenticalNoCov(*resultRef, 1e-4, 1e-3, /*verbose=*/true)); + model.expectProductUnderflows(); + EXPECT_EQ(RooAbsReal::numEvalErrors(), 0); } +INSTANTIATE_TEST_SUITE_P(RooProdPdf, ManyFactorsGradientTest, + testing::Combine(manyFactorsKinds, testing::Values(RooFit::EvalBackend::Codegen())), + [](testing::TestParamInfo const ¶mInfo) { + return std::get<0>(paramInfo.param); + }); +#endif + /// This test covers a potential problem with the custom normalization ranges /// of PDFs: the value of a RooProdPdf should not depend on the normalization /// ranges of its clients. TODO: this is still a problem! It needs to be fixed, From 30fb32cc66c1615203ab10565a09ae49161d614d Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 20 Nov 2026 21:42:24 +0000 Subject: [PATCH 6/6] [RF] Fix NegativeNormIntegralBadness test with the legacy backend On the 6.42 branch, the test also runs with the legacy backend, which doesn't exist on master anymore. Creating a legacy likelihood resets the evaluation error logging mode to PrintErrors, so the errors were printed instead of collected and the test counted none. Set the logging mode after creating the likelihood, and expect the additional error that the legacy likelihood logs for its own NaN value. Assisted-by: claude-code:claude-opus-5-5 --- roofit/roofitcore/test/testNaNPacker.cxx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/roofit/roofitcore/test/testNaNPacker.cxx b/roofit/roofitcore/test/testNaNPacker.cxx index 0fa543157d332..0249c0eee0f73 100644 --- a/roofit/roofitcore/test/testNaNPacker.cxx +++ b/roofit/roofitcore/test/testNaNPacker.cxx @@ -307,8 +307,6 @@ TEST(RooNaNPacker, CudaEvalErrorParity) TEST_P(TestForDifferentBackends, NegativeNormIntegralBadness) { const auto prevErrorLoggingMode = RooAbsReal::evalErrorLoggingMode(); - RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); - RooAbsReal::clearEvalErrorLog(); RooWorkspace ws; ws.factory("Gaussian::gauss(x[0, -10, 10], mean[0, -5, 5], sigma[1, -5, 5])"); @@ -320,6 +318,12 @@ TEST_P(TestForDifferentBackends, NegativeNormIntegralBadness) } std::unique_ptr nll{ws.pdf("gauss")->createNLL(data, _evalBackend)}; + + // Set the logging mode only after creating the likelihood, because the + // legacy backend resets it to PrintErrors. + RooAbsReal::setEvalErrorLoggingMode(RooAbsReal::CollectErrors); + RooAbsReal::clearEvalErrorLog(); + ws.var("sigma")->setVal(-1.0); const double val = nll->getVal(); @@ -327,7 +331,9 @@ TEST_P(TestForDifferentBackends, NegativeNormIntegralBadness) EXPECT_TRUE(RooNaNPacker::isNaNWithPayload(val)); EXPECT_NEAR(RooNaNPacker::unpackNaN(val), 3 * std::sqrt(2 * TMath::Pi()), 1e-5); // Each event logs one error for the integral and one for the likelihood. - EXPECT_EQ(RooAbsReal::numEvalErrors(), 6); + // The legacy likelihood also logs an error for its own NaN value. + const bool isLegacy = _evalBackend == RooFit::EvalBackend(RooFit::EvalBackend::Value::Legacy); + EXPECT_EQ(RooAbsReal::numEvalErrors(), isLegacy ? 7 : 6); RooAbsReal::clearEvalErrorLog(); RooAbsReal::setEvalErrorLoggingMode(prevErrorLoggingMode);