Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions roofit/batchcompute/res/RooBatchCompute.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,18 +97,21 @@ enum Computer {
Gamma,
GaussModelExpBasis,
Gaussian,
LogGaussian,
Identity,
Johnson,
Landau,
Lognormal,
LognormalStandard,
NegativeLogarithms,
NormalizedPdf,
LogNormalizedPdf,
Novosibirsk,
Poisson,
Polynomial,
Power,
ProdPdf,
LogProdPdf,
Ratio,
TruthModelExpBasis,
TruthModelSinBasis,
Expand Down Expand Up @@ -180,18 +183,20 @@ class RooBatchComputeInterface {
/// Compute the values for a batch of events.
///
/// The extra args (the last parameter) are read-only inputs for all
/// computers except `NormalizedPdf`, which uses them as output parameters
/// for its evaluation error counters. In the CUDA implementation, these
/// outputs are read back from the device *asynchronously*: they only
/// arrive in the caller's span with the next synchronizeCudaStream() call
/// on the stream of the passed config. The memory backing the extra args
/// of a `NormalizedPdf` call must therefore stay valid until that
/// synchronization, so it must not live on the caller's stack.
/// computers except `NormalizedPdf` and `LogNormalizedPdf`, which use the
/// first three as output parameters for their evaluation error counters.
/// In the CUDA implementation, these outputs are read back from the device
/// *asynchronously*: they only arrive in the caller's span with the next
/// synchronizeCudaStream() call on the stream of the passed config. The
/// memory backing the extra args of such a call must therefore stay valid
/// until that synchronization, so it must not live on the caller's stack.
virtual void compute(Config const &cfg, Computer, std::span<double> output, VarSpan, ArgSpan) = 0;

virtual double reduceSum(Config const &cfg, InputArr input, size_t n) = 0;
/// Reduce the probabilities to the negative log-likelihood. With
/// `logProbas`, the probabilities are given as logarithms.
virtual ReduceNLLOutput reduceNLL(Config const &cfg, std::span<const double> probas, std::span<const double> weights,
std::span<const double> offsetProbas) = 0;
std::span<const double> offsetProbas, bool logProbas) = 0;

virtual Architecture architecture() const = 0;
virtual std::string architectureName() const = 0;
Expand Down Expand Up @@ -246,10 +251,10 @@ inline double reduceSum(Config cfg, InputArr input, size_t n)
}

inline ReduceNLLOutput reduceNLL(Config cfg, std::span<const double> probas, std::span<const double> weights,
std::span<const double> offsetProbas)
std::span<const double> offsetProbas, bool logProbas = false)
{
auto dispatch = cfg.useCuda() ? dispatchCUDA : dispatchCPU;
return dispatch->reduceNLL(cfg, probas, weights, offsetProbas);
return dispatch->reduceNLL(cfg, probas, weights, offsetProbas, logProbas);
}

std::string getBatchComputeChoice();
Expand Down
205 changes: 188 additions & 17 deletions roofit/batchcompute/src/ComputeFunctions.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ of performance, maximum memory coalescing. For more details, see

#include <RooHeterogeneousMath.h>

#include <algorithm>
#include <limits>
#include <vector>

#ifdef __CUDACC__
Expand Down Expand Up @@ -465,6 +467,18 @@ __rooglobal__ void computeGaussian(Batches &batches)
}
}

__rooglobal__ void computeLogGaussian(Batches &batches)
{
auto x = batches.args[0];
auto mean = batches.args[1];
auto sigma = batches.args[2];
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
const double arg = x[i] - mean[i];
const double halfBySigmaSq = -0.5 / (sigma[i] * sigma[i]);
batches.output[i] = arg * arg * halfBySigmaSq;
}
}

__rooglobal__ void computeIdentity(Batches &batches)
{
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
Expand Down Expand Up @@ -630,6 +644,27 @@ __rooglobal__ void computeLognormalStandard(Batches &batches)
}
}

/// Add the evaluation error counts of the normalization compute functions to
/// the first three extra args.
__roodevice__ inline void addEvalErrorCounts(Batches &batches, int nType0, int nType1, int nType2)
{
// The counters live in memory that is shared between all threads in the
// CUDA case, so they need to be accumulated atomically there. On the CPU,
// every task works on its own copy of the extra args.
#ifdef __CUDACC__
if (nType0 > 0)
atomicAdd(&batches.extra[0], double(nType0));
if (nType1 > 0)
atomicAdd(&batches.extra[1], double(nType1));
if (nType2 > 0)
atomicAdd(&batches.extra[2], double(nType2));
#else
batches.extra[0] = batches.extra[0] + nType0;
batches.extra[1] = batches.extra[1] + nType1;
batches.extra[2] = batches.extra[2] + nType2;
#endif
}

__rooglobal__ void computeNormalizedPdf(Batches &batches)
{
auto rawVal = batches.args[0];
Expand Down Expand Up @@ -660,23 +695,80 @@ __rooglobal__ void computeNormalizedPdf(Batches &batches)
batches.output[i] = out;
}

// The counters live in memory that is shared between all threads in the
// CUDA case, so they need to be accumulated atomically there. Note that
// the CPU branch below is only safe because the CPU implementation runs
// single-threaded: with implicit multi-threading, the workers would share
// this memory as well and would also need atomic accumulation.
#ifdef __CUDACC__
if (nEvalErrorsType0 > 0)
atomicAdd(&batches.extra[0], double(nEvalErrorsType0));
if (nEvalErrorsType1 > 0)
atomicAdd(&batches.extra[1], double(nEvalErrorsType1));
if (nEvalErrorsType2 > 0)
atomicAdd(&batches.extra[2], double(nEvalErrorsType2));
#else
batches.extra[0] = batches.extra[0] + nEvalErrorsType0;
batches.extra[1] = batches.extra[1] + nEvalErrorsType1;
batches.extra[2] = batches.extra[2] + nEvalErrorsType2;
#endif
addEvalErrorCounts(batches, nEvalErrorsType0, nEvalErrorsType1, nEvalErrorsType2);
}

/// Logarithm of the normalized pdf for one event, with the same evaluation
/// errors as computeNormalizedPdf().
template <bool NumIsLog>
__roodevice__ inline double logNormalizedPdf(double num, double den, double logDen, int *nEvalErrors)
{
const bool numIsZero = NumIsLog ? num == -INFINITY : num == 0.;
const bool numIsNegative = !NumIsLog && num < 0.;
if (den < 0. || (den == 0. && !numIsZero)) {
nEvalErrors[0]++;
return RooNaNPacker::packFloatIntoNaN(-den + (numIsNegative ? -num : 0.));
}
if (numIsNegative) {
nEvalErrors[1]++;
return RooNaNPacker::packFloatIntoNaN(-num);
}
if (std::isnan(num)) {
nEvalErrors[2]++;
return num;
}
return numIsZero ? -INFINITY : (NumIsLog ? num : std::log(num)) - logDen;
}

/// The input modes are template parameters, so that no logarithms are
/// evaluated speculatively.
template <bool NumIsLog, bool ScalarDen>
__roodevice__ inline void computeLogNormalizedPdfImpl(Batches &batches)
{
auto rawVal = batches.args[0];
auto normVal = batches.args[1];
const double scalarLogDen = ScalarDen ? std::log(normVal[0]) : 0.;
int nEvalErrors[3] = {0, 0, 0};

// Loop without branches for the regular events, which the compiler can
// vectorize if no logarithms are taken. The other events, with evaluation
// errors or zero pdf values, are recomputed afterwards.
auto isRegular = [&](std::size_t i) {
return normVal[i] > 0. && (NumIsLog ? rawVal[i] > -INFINITY : rawVal[i] > 0.);
};
bool allRegular = true;
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
const double logNum = NumIsLog ? rawVal[i] : std::log(rawVal[i]);
batches.output[i] = logNum - (ScalarDen ? scalarLogDen : std::log(normVal[i]));
allRegular &= isRegular(i);
}
if (!allRegular) {
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
if (!isRegular(i)) {
const double logDen = ScalarDen ? scalarLogDen : std::log(normVal[i]);
batches.output[i] = logNormalizedPdf<NumIsLog>(rawVal[i], normVal[i], logDen, nEvalErrors);
}
}
}

addEvalErrorCounts(batches, nEvalErrors[0], nEvalErrors[1], nEvalErrors[2]);
}

/// Like computeNormalizedPdf(), but outputs the logarithm. The pdf values are
/// given as logarithms if `extra[3]` is nonzero.
__rooglobal__ void computeLogNormalizedPdf(Batches &batches)
{
const bool numIsLog = batches.extra[3] != 0.;
const bool scalarDen = !batches.args[1]._isVector;
if (numIsLog && scalarDen) {
computeLogNormalizedPdfImpl<true, true>(batches);
} else if (numIsLog) {
computeLogNormalizedPdfImpl<true, false>(batches);
} else if (scalarDen) {
computeLogNormalizedPdfImpl<false, true>(batches);
} else {
computeLogNormalizedPdfImpl<false, false>(batches);
}
}

/* TMath::ASinH(x) needs to be replaced with ln( x + sqrt(x^2+1))
Expand Down Expand Up @@ -785,6 +877,82 @@ __rooglobal__ void computeProdPdf(Batches &batches)
}
}

/// Logarithm of the product of the factors [begin, end) for event `i`. The
/// binary exponent is kept separate, so that the product can't underflow.
__roodevice__ inline double logOfProduct(Batches const &batches, std::size_t begin, std::size_t end, std::size_t i)
{
constexpr double ln2 = 0.693147180559945309417232121458176568;
// The mantissa stays within [2^-256, 2^256] and the factors are brought
// into [2^-512, 2^512], so their product is always a normal double.
double mantissa = 1.0;
int exponent = 0;
for (std::size_t k = begin; k < end; ++k) {
double factor = batches.args[k][i];
int e = 0;
if (std::abs(factor) < 0x1p-512 || std::abs(factor) > 0x1p512) {
factor = std::frexp(factor, &e);
exponent += e;
}
mantissa *= factor;
if (std::abs(mantissa) < 0x1p-256 || std::abs(mantissa) > 0x1p256) {
mantissa = std::frexp(mantissa, &e);
exponent += e;
}
}
if (std::isnan(mantissa)) {
return mantissa; // with the payload of the NaN factor
}
// Pack a negative product into a NaN, like computeNormalizedPdf does.
return mantissa < 0. ? RooNaNPacker::packFloatIntoNaN(-std::ldexp(mantissa, exponent))
: std::log(mantissa) + exponent * ln2;
}

/// Logarithm of a product. The first `extra[0]` factors are given as
/// logarithms and are summed, the others are multiplied.
__rooglobal__ void computeLogProdPdf(Batches &batches)
{
const std::size_t nLogFactors = batches.extra[0];
const std::size_t nFactors = batches.nBatches;
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
batches.output[i] = 0.0;
}
for (std::size_t k = 0; k < nLogFactors; ++k) {
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
batches.output[i] += batches.args[k][i];
}
}
if (nLogFactors == nFactors) {
return;
}
#ifdef __CUDACC__
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
batches.output[i] += logOfProduct(batches, nLogFactors, nFactors, i);
}
#else
// Multiply in a vectorizable loop, keeping track of the smallest partial
// product. If that is a normal number and the product didn't overflow,
// no precision was lost. Otherwise, the event is recomputed with
// logOfProduct().
double prod[bufferSize];
double minProd[bufferSize];
std::fill_n(prod, batches.nEvents, 1.0);
std::fill_n(minProd, batches.nEvents, 1.0);
for (std::size_t k = nLogFactors; k < nFactors; ++k) {
for (std::size_t i = 0; i < batches.nEvents; ++i) {
prod[i] *= batches.args[k][i];
minProd[i] = std::min(minProd[i], prod[i]);
}
}
for (std::size_t i = 0; i < batches.nEvents; ++i) {
if (minProd[i] >= std::numeric_limits<double>::min() && prod[i] <= std::numeric_limits<double>::max()) {
batches.output[i] += std::log(prod[i]);
} else {
batches.output[i] += logOfProduct(batches, nLogFactors, nFactors, i);
}
}
#endif
}

__rooglobal__ void computeRatio(Batches &batches)
{
for (size_t i = BEGIN; i < batches.nEvents; i += STEP) {
Expand Down Expand Up @@ -944,18 +1112,21 @@ std::vector<void (*)(Batches &)> getFunctions()
computeGamma,
computeGaussModelExpBasis,
computeGaussian,
computeLogGaussian,
computeIdentity,
computeJohnson,
computeLandau,
computeLognormal,
computeLognormalStandard,
computeNegativeLogarithms,
computeNormalizedPdf,
computeLogNormalizedPdf,
computeNovosibirsk,
computePoisson,
computePolynomial,
computePower,
computeProdPdf,
computeLogProdPdf,
computeRatio,
computeTruthModelExpBasis,
computeTruthModelSinBasis,
Expand Down
Loading
Loading