From d3ccabefbd43f6e1f6953fe520e48275504723d1 Mon Sep 17 00:00:00 2001 From: Brian Harrington Date: Thu, 10 Oct 2024 16:43:25 -0500 Subject: [PATCH] api: percentile approximations from doubles Update the buckets helper to allow for computing the approximations using floating point counts. Avoids the need for conversion to integer which can be tricky to do correctly in some cases. --- .../api/histogram/PercentileBuckets.java | 85 ++++++++++++++++++- .../api/histogram/PercentileBucketsTest.java | 42 ++++++++- 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/spectator-api/src/main/java/com/netflix/spectator/api/histogram/PercentileBuckets.java b/spectator-api/src/main/java/com/netflix/spectator/api/histogram/PercentileBuckets.java index 8ed6d1dc1..891b13a07 100644 --- a/spectator-api/src/main/java/com/netflix/spectator/api/histogram/PercentileBuckets.java +++ b/spectator-api/src/main/java/com/netflix/spectator/api/histogram/PercentileBuckets.java @@ -1,5 +1,5 @@ /* - * Copyright 2014-2019 Netflix, Inc. + * Copyright 2014-2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -125,7 +125,10 @@ public static void percentiles(long[] counts, double[] pcts, double[] results) { long nextB = BUCKET_VALUES[i]; while (pctIdx < pcts.length && nextP >= pcts[pctIdx]) { double f = (pcts[pctIdx] - prevP) / (nextP - prevP); - results[pctIdx] = f * (nextB - prevB) + prevB; + if (Double.isNaN(f)) + results[pctIdx] = 0.0; + else + results[pctIdx] = f * (nextB - prevB) + prevB; ++pctIdx; } if (pctIdx >= pcts.length) break; @@ -161,6 +164,84 @@ public static double percentile(long[] counts, double p) { return results[0]; } + /** + * Compute a set of percentiles based on the counts for the buckets. + * + * @param counts + * Counts for each of the buckets. The values should be a non-negative finite double + * indicating the relative amount for that bucket. The size must be the same as + * {@link #length()} and the positions must correspond to the positions of the bucket values. + * @param pcts + * Array with the requested percentile values. The length must be at least 1 and the + * array should be sorted. Each value, {@code v}, should adhere to {@code 0.0 <= v <= 100.0}. + * @param results + * The calculated percentile values will be written to the results array. It should have the + * same length as {@code pcts}. + */ + public static void percentiles(double[] counts, double[] pcts, double[] results) { + Preconditions.checkArg(counts.length == BUCKET_VALUES.length, + "counts is not the same size as buckets array"); + Preconditions.checkArg(pcts.length > 0, "pct array cannot be empty"); + Preconditions.checkArg(pcts.length == results.length, + "pcts is not the same size as results array"); + + double total = 0.0; + for (double c : counts) { + if (c > 0.0 && Double.isFinite(c)) + total += c; + } + + int pctIdx = 0; + + double prev = 0.0; + double prevP = 0.0; + long prevB = 0; + for (int i = 0; i < BUCKET_VALUES.length; ++i) { + double next = prev + counts[i]; + double nextP = 100.0 * next / total; + long nextB = BUCKET_VALUES[i]; + while (pctIdx < pcts.length && nextP >= pcts[pctIdx]) { + double f = (pcts[pctIdx] - prevP) / (nextP - prevP); + if (Double.isNaN(f)) + results[pctIdx] = 0.0; + else + results[pctIdx] = f * (nextB - prevB) + prevB; + ++pctIdx; + } + if (pctIdx >= pcts.length) break; + prev = next; + prevP = nextP; + prevB = nextB; + } + + double nextP = 100.0; + long nextB = Long.MAX_VALUE; + while (pctIdx < pcts.length) { + double f = (pcts[pctIdx] - prevP) / (nextP - prevP); + results[pctIdx] = f * (nextB - prevB) + prevB; + ++pctIdx; + } + } + + /** + * Compute a percentile based on the counts for the buckets. + * + * @param counts + * Counts for each of the buckets. The values should be a non-negative finite double + * indicating the relative amount for that bucket. The size must be the same as + * {@link #length()} and the positions must correspond to the positions of the bucket values. + * @param p + * Percentile to compute, the value should be {@code 0.0 <= p <= 100.0}. + * @return + * The calculated percentile value. + */ + public static double percentile(double[] counts, double p) { + double[] pcts = {p}; + double[] results = new double[1]; + percentiles(counts, pcts, results); + return results[0]; + } + // Number of positions of base-2 digits to shift when iterating over the long space. private static final int DIGITS = 2; diff --git a/spectator-api/src/test/java/com/netflix/spectator/api/histogram/PercentileBucketsTest.java b/spectator-api/src/test/java/com/netflix/spectator/api/histogram/PercentileBucketsTest.java index d8a242ea8..60a01d450 100644 --- a/spectator-api/src/test/java/com/netflix/spectator/api/histogram/PercentileBucketsTest.java +++ b/spectator-api/src/test/java/com/netflix/spectator/api/histogram/PercentileBucketsTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2014-2019 Netflix, Inc. + * Copyright 2014-2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -126,4 +126,44 @@ public void percentile() { Assertions.assertEquals(expected, PercentileBuckets.percentile(counts, pct), threshold); } } + + @Test + public void percentilesDouble() { + double[] counts = new double[PercentileBuckets.length()]; + for (int i = 0; i < 100_000; ++i) { + // simulate it as a rate per minute + counts[PercentileBuckets.indexOf(i)] += 1.0 / 60.0; + } + + double[] pcts = new double[] {0.0, 25.0, 50.0, 75.0, 90.0, 95.0, 98.0, 99.0, 99.5, 100.0}; + double[] results = new double[pcts.length]; + + PercentileBuckets.percentiles(counts, pcts, results); + + double[] expected = new double[] {0.0, 25e3, 50e3, 75e3, 90e3, 95e3, 98e3, 99e3, 99.5e3, 100e3}; + double threshold = 0.1 * 100_000; // quick check, should be within 10% of total + Assertions.assertArrayEquals(expected, results, threshold); + + // Further check each value is within 10% of actual percentile + for (int i = 0 ; i < results.length; ++i) { + threshold = 0.1 * expected[i] + 1e-12; + Assertions.assertEquals(expected[i], results[i], threshold); + } + } + + @Test + public void percentileDouble() { + double[] counts = new double[PercentileBuckets.length()]; + for (int i = 0; i < 100_000; ++i) { + // simulate it as a rate per minute + counts[PercentileBuckets.indexOf(i)] += 1.0 / 60.0; + } + + double[] pcts = new double[] {0.0, 25.0, 50.0, 75.0, 90.0, 95.0, 98.0, 99.0, 99.5, 100.0}; + for (double pct : pcts) { + double expected = pct * 1e3; + double threshold = 0.1 * expected + 1e-12; + Assertions.assertEquals(expected, PercentileBuckets.percentile(counts, pct), threshold); + } + } }