majordomus_cli/bench/
stats.rs

1//! Latency statistics of one target: the samples reduced to the numbers a baseline is
2//! compared on. Durations are nanoseconds on a monotonic clock, reported in
3//! microseconds.
4
5use std::time::Duration;
6
7use schemars::JsonSchema;
8use serde::{Deserialize, Serialize};
9
10/// The statistics of one set of samples, in microseconds.
11#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
12pub struct Statistics {
13    /// How many samples.
14    pub samples: usize,
15    /// The smallest.
16    pub min_us: f64,
17    /// The median.
18    pub p50_us: f64,
19    /// The 90th percentile.
20    pub p90_us: f64,
21    /// The 95th percentile.
22    pub p95_us: f64,
23    /// The 99th percentile.
24    pub p99_us: f64,
25    /// The largest.
26    pub max_us: f64,
27    /// The mean.
28    pub mean_us: f64,
29    /// The population standard deviation.
30    pub stddev_us: f64,
31}
32
33impl Statistics {
34    /// Reduce samples. An empty set is all zeros with `samples: 0`.
35    ///
36    /// ```
37    /// use std::time::Duration;
38    /// use majordomus_cli::bench::Statistics;
39    /// let s = Statistics::of(&(1..=100).map(Duration::from_micros).collect::<Vec<_>>());
40    /// assert_eq!(s.samples, 100);
41    /// assert_eq!(s.min_us, 1.0);
42    /// assert_eq!(s.max_us, 100.0);
43    /// assert_eq!(s.p50_us, 50.0);
44    /// assert_eq!(s.p99_us, 99.0);
45    /// assert!((s.mean_us - 50.5).abs() < 1e-9);
46    /// ```
47    pub fn of(samples: &[Duration]) -> Self {
48        if samples.is_empty() {
49            return Statistics {
50                samples: 0,
51                min_us: 0.0,
52                p50_us: 0.0,
53                p90_us: 0.0,
54                p95_us: 0.0,
55                p99_us: 0.0,
56                max_us: 0.0,
57                mean_us: 0.0,
58                stddev_us: 0.0,
59            };
60        }
61        let mut us: Vec<f64> = samples
62            .iter()
63            .map(|d| d.as_nanos() as f64 / 1000.0)
64            .collect();
65        // Ascending by value, and deliberately not the canonical order: the percentiles
66        // below index into this vector by rank, so the sequence is the statistic's own
67        // definition and nothing renders it.
68        us.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
69        let n = us.len();
70        let pct = |p: f64| -> f64 {
71            // nearest-rank: the smallest value below which p percent of the samples fall
72            let rank = ((p / 100.0) * n as f64).ceil() as usize;
73            us[rank.clamp(1, n) - 1]
74        };
75        let mean = us.iter().sum::<f64>() / n as f64;
76        let var = us.iter().map(|x| (x - mean) * (x - mean)).sum::<f64>() / n as f64;
77        Statistics {
78            samples: n,
79            min_us: us[0],
80            p50_us: pct(50.0),
81            p90_us: pct(90.0),
82            p95_us: pct(95.0),
83            p99_us: pct(99.0),
84            max_us: us[n - 1],
85            mean_us: mean,
86            stddev_us: var.sqrt(),
87        }
88    }
89
90    /// How many samples are certainly at or above a metric's value: the samples a verdict
91    /// on that metric rests on. A nearest-rank percentile of rank `r` has `n - r + 1` samples
92    /// at or above it; the maximum has one, the minimum all of them; the mean is an
93    /// aggregate of every sample and one outlier can carry it, so it counts as one. `None`
94    /// for a metric this does not know.
95    ///
96    /// A metric over its threshold is a regression only when at least two samples agree:
97    /// the median of one sample, or the p99 of twenty, is the single slowest sample, which
98    /// one scheduler hiccup on a loaded runner decides.
99    ///
100    /// ```
101    /// use std::time::Duration;
102    /// use majordomus_cli::bench::Statistics;
103    /// let one = Statistics::of(&[Duration::from_millis(23)]);
104    /// assert_eq!(one.samples_at_or_above("p50"), Some(1));
105    /// let five = Statistics::of(&vec![Duration::from_millis(9); 5]);
106    /// assert_eq!(five.samples_at_or_above("p50"), Some(3));
107    /// assert_eq!(five.samples_at_or_above("p99"), Some(1));
108    /// assert_eq!(five.samples_at_or_above("min"), Some(5));
109    /// assert_eq!(five.samples_at_or_above("mean"), Some(1));
110    /// assert_eq!(five.samples_at_or_above("p42"), None);
111    /// ```
112    pub fn samples_at_or_above(&self, metric: &str) -> Option<usize> {
113        let n = self.samples;
114        if n == 0 {
115            return self.metric(metric).map(|_| 0);
116        }
117        let above_rank = |p: f64| {
118            let rank = ((p / 100.0) * n as f64).ceil() as usize;
119            n - rank.clamp(1, n) + 1
120        };
121        Some(match metric {
122            "min" => n,
123            "p50" => above_rank(50.0),
124            "p90" => above_rank(90.0),
125            "p95" => above_rank(95.0),
126            "p99" => above_rank(99.0),
127            "max" | "mean" => 1,
128            _ => return None,
129        })
130    }
131
132    /// One metric by name (`p50`, `p95`, `p99`, `mean`, `max`, `min`, `p90`).
133    pub fn metric(&self, name: &str) -> Option<f64> {
134        Some(match name {
135            "min" => self.min_us,
136            "p50" => self.p50_us,
137            "p90" => self.p90_us,
138            "p95" => self.p95_us,
139            "p99" => self.p99_us,
140            "max" => self.max_us,
141            "mean" => self.mean_us,
142            _ => return None,
143        })
144    }
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150
151    fn micros(values: impl IntoIterator<Item = u64>) -> Vec<Duration> {
152        values.into_iter().map(Duration::from_micros).collect()
153    }
154
155    #[test]
156    fn an_empty_set_counts_zero_for_known_metrics_and_none_for_unknown() {
157        let empty = Statistics::of(&[]);
158        assert_eq!(empty.samples, 0);
159        for metric in ["min", "p50", "p90", "p95", "p99", "max", "mean"] {
160            assert_eq!(empty.samples_at_or_above(metric), Some(0), "{metric}");
161        }
162        assert_eq!(empty.samples_at_or_above("p42"), None);
163    }
164
165    #[test]
166    fn every_metric_names_the_samples_its_verdict_rests_on() {
167        let twenty = Statistics::of(&micros(1..=20));
168        assert_eq!(twenty.samples_at_or_above("min"), Some(20));
169        assert_eq!(twenty.samples_at_or_above("p50"), Some(11));
170        assert_eq!(twenty.samples_at_or_above("p90"), Some(3));
171        assert_eq!(twenty.samples_at_or_above("p95"), Some(2));
172        assert_eq!(twenty.samples_at_or_above("p99"), Some(1));
173        assert_eq!(twenty.samples_at_or_above("max"), Some(1));
174        assert_eq!(twenty.samples_at_or_above("mean"), Some(1));
175        assert_eq!(twenty.samples_at_or_above("stddev"), None);
176    }
177
178    #[test]
179    fn one_sample_rests_every_percentile_on_that_sample() {
180        let one = Statistics::of(&micros([23_000]));
181        for metric in ["min", "p50", "p90", "p95", "p99", "max", "mean"] {
182            assert_eq!(one.samples_at_or_above(metric), Some(1), "{metric}");
183        }
184        assert_eq!(one.metric("p50"), Some(23_000.0));
185    }
186
187    #[test]
188    fn metric_reads_each_named_field() {
189        let s = Statistics::of(&micros(1..=100));
190        assert_eq!(s.metric("min"), Some(1.0));
191        assert_eq!(s.metric("p50"), Some(50.0));
192        assert_eq!(s.metric("p90"), Some(90.0));
193        assert_eq!(s.metric("p95"), Some(95.0));
194        assert_eq!(s.metric("p99"), Some(99.0));
195        assert_eq!(s.metric("max"), Some(100.0));
196        assert_eq!(s.metric("mean"), Some(50.5));
197        assert_eq!(s.metric("p42"), None);
198    }
199}
majordomus_cli/bench/stats.rs

majordomus_cli/bench/
stats.rs
