From 0b1c731b78db5aed7bdb4d0b89fcf657fb837655 Mon Sep 17 00:00:00 2001 From: Mahammad Insaf Date: Fri, 6 Nov 2026 23:23:17 +0530 Subject: [PATCH 1/2] fix percentile reordering data in DescriptiveStatistics and StatUtils The Commons Statistics Quantile used by Statistics.Percentile and StatUtils was built with withCopy(false), so it partially sorts the array it is given. DescriptiveStatistics passes its stored window, so getPercentile (and toString) reordered the values: getValues no longer returned them in insertion order, and a rolling window then discarded the smallest value instead of the oldest. StatUtils.percentile reordered the caller's array. The Percentile class replaced in MATH-1671 worked on a copy. --- .../commons/math4/legacy/stat/StatUtils.java | 2 +- .../legacy/stat/descriptive/Statistics.java | 2 +- .../commons/math4/legacy/stat/StatUtilsTest.java | 8 ++++++++ .../descriptive/DescriptiveStatisticsTest.java | 16 ++++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java index 841c465648..b40347049b 100644 --- a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java +++ b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java @@ -41,7 +41,7 @@ public final class StatUtils { /** percentile. */ - private static final Quantile QUANTILE = Quantile.withDefaults().with(EstimationMethod.HF6).withCopy(false); + private static final Quantile QUANTILE = Quantile.withDefaults().with(EstimationMethod.HF6).withCopy(true); /** * Private Constructor. diff --git a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/Statistics.java b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/Statistics.java index 9445ea4c37..a097e7df0a 100644 --- a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/Statistics.java +++ b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/Statistics.java @@ -316,7 +316,7 @@ static Sum getInstance() { static final class Percentile implements UnivariateStatistic { /** Delegate percentile implementation. */ private static final Quantile QUANTILE = - Quantile.withDefaults().with(EstimationMethod.HF6).withCopy(false); + Quantile.withDefaults().with(EstimationMethod.HF6).withCopy(true); /** Probability. */ private double p; diff --git a/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/StatUtilsTest.java b/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/StatUtilsTest.java index b5a8dcd5c8..5bc4bfb657 100644 --- a/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/StatUtilsTest.java +++ b/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/StatUtilsTest.java @@ -456,6 +456,14 @@ public void testPercentile() { TestUtils.assertEquals(2.5, StatUtils.percentile(x, 1, 3, 62.5), TOLERANCE); } + @Test + public void testPercentileDoesNotModifyInput() { + final double[] x = {THREE, ONE, TWO, TWO}; + StatUtils.percentile(x, 50); + StatUtils.percentile(x, 1, 3, 50); + Assert.assertArrayEquals(new double[] {THREE, ONE, TWO, TWO}, x, 0); + } + @Test public void testDifferenceStats() { double sample1[] = {1d, 2d, 3d, 4d}; diff --git a/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatisticsTest.java b/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatisticsTest.java index c47e7f57f0..0fdfba57c9 100644 --- a/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatisticsTest.java +++ b/commons-math-legacy/src/test/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatisticsTest.java @@ -125,6 +125,22 @@ public void testGetValues() { Assert.assertEquals(12.0, stats.getElement(88), 1.0e-10); } + @Test + public void testPercentileDoesNotReorderValues() { + DescriptiveStatistics stats = createDescriptiveStatistics(); + stats.setWindowSize(5); + for (double v : new double[] {5, 4, 3, 2, 1}) { + stats.addValue(v); + } + Assert.assertEquals(3, stats.getPercentile(50), 0); + // toString() reports the median + Assert.assertNotNull(stats.toString()); + Assert.assertArrayEquals(new double[] {5, 4, 3, 2, 1}, stats.getValues(), 0); + // The oldest value is discarded + stats.addValue(100); + Assert.assertArrayEquals(new double[] {4, 3, 2, 1, 100}, stats.getValues(), 0); + } + @Test public void testQuadraticMean() { final double[] values = { 1.2, 3.4, 5.6, 7.89 }; From ba1aa9ea6bdba351e4118ea53dcfbc488197b034 Mon Sep 17 00:00:00 2001 From: Mahammad Insaf Date: Tue, 17 Nov 2026 11:32:06 +0530 Subject: [PATCH 2/2] document the per-call copy in getPercentile and StatUtils.percentile --- .../commons/math4/legacy/stat/StatUtils.java | 15 +++++++++++++++ .../stat/descriptive/DescriptiveStatistics.java | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java index b40347049b..824d0567d9 100644 --- a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java +++ b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/StatUtils.java @@ -477,6 +477,13 @@ public static double min(final double[] values, final int begin, final int lengt * *

* See Commons Statistics {@link Quantile} for a description of the percentile estimation algorithm used. + *

+ * The percentile is computed on a copy of the input so the order of + * values is not changed. The copy is made on every call. To + * compute several percentiles of the same data use a single call of + * {@link Quantile#evaluate(double[], double...)} with all the probabilities + * (scaled from 0 - 1), using {@link EstimationMethod#HF6} to match this method, + * and {@link Quantile#withCopy(boolean) withCopy(false)} if the input may be reordered. * * @param values input array of values * @param p the percentile value to compute @@ -504,6 +511,14 @@ public static double percentile(final double[] values, final double p) throws Ma * *

* See Commons Statistics {@link Quantile} for a description of the percentile estimation algorithm used. + *

+ * The percentile is computed on a copy of the specified range so the order of + * values is not changed. The copy is made on every call. To + * compute several percentiles of the same data use a single call of + * {@link Quantile#evaluateRange(double[], int, int, double...)} with all the + * probabilities (scaled from 0 - 1), using {@link EstimationMethod#HF6} to match + * this method, and {@link Quantile#withCopy(boolean) withCopy(false)} if the input + * may be reordered. * * @param values array of input values * @param p the percentile to compute diff --git a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatistics.java b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatistics.java index 518488495c..b3cbff667b 100644 --- a/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatistics.java +++ b/commons-math-legacy/src/main/java/org/apache/commons/math4/legacy/stat/descriptive/DescriptiveStatistics.java @@ -437,6 +437,23 @@ public double getElement(int index) { *

  • at least one value must be stored (returns Double.NaN * otherwise)
  • * + *

    + * The default implementation computes the percentile on a copy of the + * stored values so the order returned by {@link #getValues()} is preserved. + * The copy and the selection are repeated on every call, even if the stored + * values have not changed. To compute several percentiles of the same data, + * get the values once and pass all the probabilities (scaled from 0 - 1) to + * a single call of Commons Statistics + * {@link org.apache.commons.statistics.descriptive.Quantile#evaluate(double[], double...) + * Quantile.evaluate}. The same estimation as this method is obtained with: + *

    +     * Quantile.withDefaults()
    +     *         .with(Quantile.EstimationMethod.HF6)
    +     *         .withCopy(false)
    +     *         .evaluate(stats.getValues(), 0.25, 0.5, 0.75);
    +     * 
    + *

    + * {@code withCopy(false)} is safe here since {@link #getValues()} returns a copy. * * @param p the requested percentile (scaled from 0 - 100) * @return An estimate for the pth percentile of the stored data