majordomus_cli/bench/
results.rs

1//! The result document: a versioned record of one benchmark run, with the provenance
2//! that makes it comparable (commit, dirty state, build profile, platform, registry
3//! fingerprint, profile) and, per target, the statistics of each cache mode measured.
4//! Local runs are written under the checkout's local half; the accepted baseline is a
5//! tracked file of the same shape, minus the machine-local fields, promoted explicitly.
6
7use std::path::{Path, PathBuf};
8
9use schemars::JsonSchema;
10use serde::{Deserialize, Serialize};
11
12use crate::error::{Error, Result};
13use crate::repository::Repository;
14
15use super::projection::TargetKind;
16use super::stats::Statistics;
17
18/// The schema of a result document.
19pub const RESULT_SCHEMA: &str = "majordomus/benchmark-result/v1";
20
21/// Where local results go, relative to the checkout-local half.
22pub const LOCAL_DIR: &str = "benchmarks";
23
24/// How the cache stood when the samples were taken.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
26#[serde(rename_all = "snake_case")]
27pub enum CacheMode {
28    /// The capability declares no cache.
29    Uncached,
30    /// The cache was cleared before every sample: the handler ran each time.
31    Cold,
32    /// The same input repeated: answered from the cache.
33    Warm,
34    /// Not a capability call.
35    NotApplicable,
36}
37
38/// One target's measurement.
39#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
40pub struct BenchmarkResult {
41    /// The target's key.
42    pub key: String,
43    /// What was measured.
44    pub kind: TargetKind,
45    /// The cache mode.
46    pub cache_mode: CacheMode,
47    /// The statistics.
48    pub stats: Statistics,
49    /// Handler invocations during the samples, when known (direct transport).
50    #[serde(skip_serializing_if = "Option::is_none")]
51    pub handler_invocations: Option<u64>,
52}
53
54/// The provenance a run carries.
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
56pub struct Provenance {
57    /// Full commit id of HEAD, when known.
58    #[serde(skip_serializing_if = "Option::is_none")]
59    pub commit: Option<String>,
60    /// Was the work tree dirty?
61    pub dirty: bool,
62    /// `debug` or `release`.
63    pub build_profile: String,
64    /// `std::env::consts::OS`.
65    pub os: String,
66    /// `std::env::consts::ARCH`.
67    pub arch: String,
68    /// The executable's version.
69    pub version: String,
70    /// The registry fingerprint of the repository measured.
71    pub registry_fingerprint: String,
72    /// The host that measured: the CPU's brand string and its logical core count
73    /// (`Apple M5 Pro/18`). Wall-clock numbers are comparable only between runs on the
74    /// same host; a baseline recorded elsewhere is reported against, never failed
75    /// against. Empty when the host could not be identified (older baselines).
76    #[serde(default)]
77    pub host: String,
78    /// The host's 1-minute load average when the run finished, two decimals (`3.41`): what
79    /// the samples were measured under. A timing verdict is a statement about a machine as
80    /// much as about a change, so every finding of a check names it. Empty when the host
81    /// does not report one (and in documents recorded before it was kept).
82    #[serde(default, skip_serializing_if = "String::is_empty")]
83    pub load: String,
84}
85
86impl Provenance {
87    /// This process, this repository.
88    pub fn of(repo: &Repository, registry_fingerprint: &str) -> Self {
89        let git = crate::git::inspect(repo.root());
90        let (commit, dirty) = match git {
91            crate::git::GitState::Available(info) => (info.head, info.working_tree == "dirty"),
92            crate::git::GitState::Unavailable { .. } => (None, true),
93        };
94        Provenance {
95            commit,
96            dirty,
97            build_profile: if cfg!(debug_assertions) {
98                "debug".into()
99            } else {
100                "release".into()
101            },
102            os: std::env::consts::OS.into(),
103            arch: std::env::consts::ARCH.into(),
104            host: host_fingerprint(),
105            load: load_average()
106                .map(|l| format!("{l:.2}"))
107                .unwrap_or_default(),
108            version: crate::VERSION.into(),
109            registry_fingerprint: registry_fingerprint.into(),
110        }
111    }
112
113    /// The platform key baselines are kept by.
114    pub fn platform(&self) -> String {
115        format!("{}-{}-{}", self.os, self.arch, self.build_profile)
116    }
117
118    /// Were these two measured on the same host? Unknown hosts never match.
119    pub fn same_host(&self, other: &Provenance) -> bool {
120        !self.host.is_empty() && self.host == other.host
121    }
122}
123
124/// The CPU brand string and the logical core count, `<brand>/<cores>`; the brand alone
125/// when the count is unknown, the os and arch when neither is (an unidentified host
126/// still gets a stable, if coarse, name, and two of them never compare as equal only
127/// when the string is empty, which this function does not return).
128pub fn host_fingerprint() -> String {
129    let cores = std::thread::available_parallelism()
130        .map(|n| n.get().to_string())
131        .unwrap_or_default();
132    let brand = cpu_brand()
133        .unwrap_or_else(|| format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH));
134    if cores.is_empty() {
135        brand
136    } else {
137        format!("{brand}/{cores}")
138    }
139}
140
141/// The host's 1-minute load average, from `getloadavg(3)`; `None` where the platform has
142/// none or the call fails.
143///
144/// ```
145/// use majordomus_cli::bench::results::load_average;
146/// if let Some(load) = load_average() {
147///     assert!(load >= 0.0 && load.is_finite());
148/// }
149/// ```
150pub fn load_average() -> Option<f64> {
151    #[cfg(unix)]
152    {
153        let mut loads = [0f64; 1];
154        // SAFETY: getloadavg writes at most `nelem` doubles into the buffer it is given,
155        // and the buffer holds exactly that many.
156        let n = unsafe { libc::getloadavg(loads.as_mut_ptr(), 1) };
157        (n == 1 && loads[0].is_finite() && loads[0] >= 0.0).then_some(loads[0])
158    }
159    #[cfg(not(unix))]
160    {
161        None
162    }
163}
164
165/// The CPU brand string: `sysctl machdep.cpu.brand_string` on macOS, the first
166/// `model name` of `/proc/cpuinfo` on Linux, nothing elsewhere.
167fn cpu_brand() -> Option<String> {
168    if cfg!(target_os = "macos") {
169        let out = std::process::Command::new("sysctl")
170            .args(["-n", "machdep.cpu.brand_string"])
171            .output()
172            .ok()?;
173        let s = String::from_utf8_lossy(&out.stdout).trim().to_string();
174        return (!s.is_empty()).then_some(s);
175    }
176    if cfg!(target_os = "linux") {
177        let text = std::fs::read_to_string("/proc/cpuinfo").ok()?;
178        return text
179            .lines()
180            .find(|l| l.starts_with("model name"))
181            .and_then(|l| l.split_once(':'))
182            .map(|(_, v)| v.trim().to_string())
183            .filter(|v| !v.is_empty());
184    }
185    None
186}
187
188/// One run.
189#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
190pub struct ResultDocument {
191    /// `majordomus/benchmark-result/v1`.
192    pub schema: String,
193    /// When the run finished, RFC 3339 UTC. Observed evidence, not a generated artifact.
194    pub finished_at: String,
195    /// The profile the run used.
196    pub profile: String,
197    /// Provenance.
198    pub provenance: Provenance,
199    /// Every measurement, by key then cache mode.
200    pub results: Vec<BenchmarkResult>,
201}
202
203impl ResultDocument {
204    /// Write under `<local>/benchmarks/<utc>-<profile>.json`; returns the path.
205    pub fn write_local(&self, repo: &Repository) -> Result<PathBuf> {
206        let dir = repo.root().join(repo.local_path()).join(LOCAL_DIR);
207        std::fs::create_dir_all(&dir).map_err(|e| Error::io(&dir, e))?;
208        let stamp = self.finished_at.replace([':', '-'], "");
209        let path = dir.join(format!("{stamp}-{}.json", self.profile));
210        write_atomic(&path, &self.render())?;
211        Ok(path)
212    }
213
214    /// Pretty JSON with a trailing newline.
215    pub fn render(&self) -> String {
216        let mut s = serde_json::to_string_pretty(self).unwrap_or_default();
217        s.push('\n');
218        s
219    }
220
221    /// One result by key and cache mode.
222    pub fn find(&self, key: &str, mode: CacheMode) -> Option<&BenchmarkResult> {
223        self.results
224            .iter()
225            .find(|r| r.key == key && r.cache_mode == mode)
226    }
227}
228
229/// Write a file through a temporary neighbour and a rename, so that a reader never sees
230/// half a file.
231pub fn write_atomic(path: &Path, content: &str) -> Result<()> {
232    let tmp = path.with_extension("json.tmp");
233    std::fs::write(&tmp, content).map_err(|e| Error::io(&tmp, e))?;
234    std::fs::rename(&tmp, path).map_err(|e| Error::io(path, e))
235}
majordomus_cli/bench/results.rs

majordomus_cli/bench/
results.rs
