majordomus_cli/bench/
coverage.rs

1//! Benchmark coverage: the denominator is generated from the registry and the
2//! projection, never written down. A required capability is covered on a transport when
3//! the projection has a target for it there (its input type produced a case in this
4//! repository); missing when it has none; waived when its policy says so, which is
5//! reported and never counted as covered. System targets are always required and always
6//! covered by construction, and listed so that the total is honest.
7
8use std::collections::BTreeMap;
9
10use schemars::JsonSchema;
11use serde::{Deserialize, Serialize};
12
13use crate::capability::{BenchmarkPolicy, Context};
14
15use super::projection::{BenchmarkProjection, Transport};
16use super::system::SystemTarget;
17
18/// The schema of the coverage document.
19pub const COVERAGE_SCHEMA: &str = "majordomus/benchmark-coverage/v1";
20
21/// Where one requirement stands.
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
23#[serde(rename_all = "snake_case")]
24pub enum CoverageState {
25    /// A target exists.
26    Covered,
27    /// Required, and no case could be produced for this repository.
28    Missing,
29    /// Waived by the descriptor, for its typed reason.
30    Waived,
31    /// Required on a precondition this repository does not meet, and without a case. Not
32    /// covered (nothing was timed) and not waived (it is timed wherever the precondition
33    /// holds); the precondition is the reason.
34    Inapplicable,
35}
36
37/// One requirement: a capability on a transport, or a system target.
38#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
39pub struct CoverageLine {
40    /// The capability id, or the system target's key.
41    pub subject: String,
42    /// The module, or `system`.
43    pub module: String,
44    /// The transport.
45    pub transport: Transport,
46    /// Where it stands.
47    pub state: CoverageState,
48    /// How many cases feed it.
49    pub cases: usize,
50    /// The waiver's reason, when waived.
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub reason: Option<String>,
53}
54
55/// Per-transport tallies.
56#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
57pub struct Tally {
58    /// Requirements.
59    pub required: usize,
60    /// Covered.
61    pub covered: usize,
62    /// Missing.
63    pub missing: usize,
64    /// Waived.
65    pub waived: usize,
66    /// Inapplicable: a precondition this repository does not meet.
67    #[serde(default)]
68    pub inapplicable: usize,
69}
70
71/// The module name this projection gives the transports' own targets, and the bucket it
72/// tallies them under. It is not a capability module and no capability module may be
73/// called it: a capability whose module were `system` would have its per-transport lines
74/// tallied here instead, and would vanish from the direct, MCP and HTTP denominators
75/// without any check saying so. The unit test `reserved_namespace_is_not_a_module` holds
76/// it shut.
77pub const SYSTEM_MODULE: &str = "system";
78
79/// The coverage document.
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
81pub struct Coverage {
82    /// `majordomus/benchmark-coverage/v1`.
83    pub schema: String,
84    /// Every requirement.
85    pub lines: Vec<CoverageLine>,
86    /// Tallies by transport name, plus `system` and `total`.
87    pub tallies: BTreeMap<String, Tally>,
88}
89
90impl Coverage {
91    /// Compute coverage for a context.
92    pub fn compute(ctx: &Context, projection: &BenchmarkProjection) -> Self {
93        let mut lines = Vec::new();
94        for c in ctx.registry.iter() {
95            if !c.kind.is_executable() || !c.stability.executable() {
96                continue;
97            }
98            let exposures = [
99                (Transport::Direct, true),
100                (
101                    Transport::Mcp,
102                    c.exposure.mcp.as_ref().is_some_and(|m| m.tool.is_some()),
103                ),
104                (Transport::Http, c.exposure.http.is_some()),
105            ];
106            for (transport, exposed) in exposures {
107                if !exposed {
108                    continue;
109                }
110                let cases = projection
111                    .of_capability(c.id.as_str())
112                    .filter(|t| t.transport() == transport)
113                    .count();
114                let (state, reason) = match c.benchmark {
115                    BenchmarkPolicy::Waived { reason } => (
116                        CoverageState::Waived,
117                        Some(
118                            serde_json::to_value(reason)
119                                .ok()
120                                .and_then(|v| v.as_str().map(str::to_string))
121                                .unwrap_or_default(),
122                        ),
123                    ),
124                    BenchmarkPolicy::Required if cases > 0 => (CoverageState::Covered, None),
125                    BenchmarkPolicy::Required => (CoverageState::Missing, None),
126                    // a case is a case: the precondition only decides what no case means
127                    BenchmarkPolicy::RequiredWhen { .. } if cases > 0 => {
128                        (CoverageState::Covered, None)
129                    }
130                    BenchmarkPolicy::RequiredWhen { precondition }
131                        if precondition.holds(&ctx.index) =>
132                    {
133                        (CoverageState::Missing, None)
134                    }
135                    BenchmarkPolicy::RequiredWhen { precondition } => (
136                        CoverageState::Inapplicable,
137                        Some(
138                            serde_json::to_value(precondition)
139                                .ok()
140                                .and_then(|v| v.as_str().map(str::to_string))
141                                .unwrap_or_default(),
142                        ),
143                    ),
144                };
145                lines.push(CoverageLine {
146                    subject: c.id.to_string(),
147                    module: c.module.as_str().to_string(),
148                    transport,
149                    state,
150                    cases,
151                    reason,
152                });
153            }
154        }
155        for s in SystemTarget::ALL {
156            lines.push(CoverageLine {
157                subject: s.key().to_string(),
158                module: SYSTEM_MODULE.into(),
159                transport: s.transport(),
160                state: if projection.targets.iter().any(|t| t.key == s.key()) {
161                    CoverageState::Covered
162                } else {
163                    CoverageState::Missing
164                },
165                cases: 1,
166                reason: None,
167            });
168        }
169        let mut tallies: BTreeMap<String, Tally> = BTreeMap::new();
170        for line in &lines {
171            // `system` is this projection's own bucket for the transports' targets, which
172            // is why no capability module may be called that: `reserved_namespace_is_not_a_module`
173            let bucket = if line.module == SYSTEM_MODULE {
174                "system".to_string()
175            } else {
176                line.transport.name().to_string()
177            };
178            for key in [bucket, "total".to_string()] {
179                let t = tallies.entry(key).or_default();
180                t.required += 1;
181                match line.state {
182                    CoverageState::Covered => t.covered += 1,
183                    CoverageState::Missing => t.missing += 1,
184                    CoverageState::Waived => t.waived += 1,
185                    CoverageState::Inapplicable => t.inapplicable += 1,
186                }
187            }
188        }
189        Coverage {
190            schema: COVERAGE_SCHEMA.into(),
191            lines,
192            tallies,
193        }
194    }
195
196    /// Nothing missing, nothing waived: the state the repository wants.
197    pub fn is_complete(&self) -> bool {
198        let total = self.tallies.get("total").cloned().unwrap_or_default();
199        total.missing == 0 && total.waived == 0
200    }
201
202    /// Nothing missing; waivers reported but not failing.
203    pub fn has_no_missing(&self) -> bool {
204        self.tallies.get("total").is_none_or(|t| t.missing == 0)
205    }
206
207    /// The human report.
208    pub fn render(&self) -> String {
209        let mut s = String::from("Benchmark coverage\n\n");
210        for (name, t) in &self.tallies {
211            if name == "total" {
212                continue;
213            }
214            s.push_str(&format!(
215                "{:<12} {:>3} / {:>3}{}\n",
216                name,
217                t.covered,
218                t.required,
219                if t.missing + t.waived > 0 {
220                    format!("   missing {} waived {}", t.missing, t.waived)
221                } else {
222                    String::new()
223                }
224            ));
225        }
226        let total = self.tallies.get("total").cloned().unwrap_or_default();
227        s.push_str(&format!(
228            "\ntotal        {:>3} / {:>3}\nmissing      {:>3}\nwaived       {:>3}\n",
229            total.covered, total.required, total.missing, total.waived
230        ));
231        if total.inapplicable > 0 {
232            s.push_str(&format!("inapplicable {:>3}\n", total.inapplicable));
233        }
234        for line in self
235            .lines
236            .iter()
237            .filter(|l| l.state != CoverageState::Covered)
238        {
239            s.push_str(&format!(
240                "{:<8} {} on {}{}\n",
241                format!("{:?}", line.state).to_uppercase(),
242                line.subject,
243                line.transport.name(),
244                line.reason
245                    .as_ref()
246                    .map(|r| format!(" ({r})"))
247                    .unwrap_or_default()
248            ));
249        }
250        s
251    }
252}
253
254#[cfg(test)]
255mod tests {
256    /// The one name this projection reserves is not taken by a capability module. Adding a
257    /// module called `system` would compile, validate, and quietly move three coverage
258    /// lines per capability into the transports' bucket; this is where that stops.
259    #[test]
260    fn reserved_namespace_is_not_a_module() {
261        for m in crate::capability::builtin::modules() {
262            assert_ne!(
263                m.id.as_str(),
264                super::SYSTEM_MODULE,
265                "the benchmark projection reserves `{}` for the transports' own targets; \
266                 a capability module of that name has its lines tallied as transport \
267                 targets and disappears from the per-transport denominators",
268                super::SYSTEM_MODULE
269            );
270        }
271    }
272}
majordomus_cli/bench/coverage.rs

majordomus_cli/bench/
coverage.rs
