majordomus_cli/quality/
report.rs

1//! Assembling one measurement: the inventory and the registry in, a [`QualityReport`] out.
2//!
3//! # One discovery, many renderings
4//!
5//! The crate is parsed once, the registry is read once, and everything downstream — the
6//! terminal rendering, the JSON, the HTTP response, the Cockpit panel, the CI annotations
7//! — is a reading of the single value this module returns. That is the same relationship
8//! every other projection in this executable has with the registry, and it is why a
9//! `--json` report and a human report can never disagree about whether the gate passed.
10//!
11//! # What it measures
12//!
13//! ```text
14//! src/**.rs  --syn-->  Inventory  --policy-->  item and module findings
15//! registry + clap tree + OpenAPI  --parity-->  operation findings
16//! ```
17//!
18//! ```
19//! use majordomus_cli::quality::report;
20//!
21//! // a crate of one module, documented, exampled and tested: the shape of a clean report
22//! let dir = tempfile::tempdir().unwrap();
23//! let src = dir.path().join("src");
24//! std::fs::create_dir_all(&src).unwrap();
25//! std::fs::write(src.join("lib.rs"), concat!(
26//!     "//! A crate that does one thing, and this header says at some length what that one\n",
27//!     "//! thing is, which is what the module policy asks of every exported module here.\n",
28//!     "//!\n",
29//!     "//! ```\n",
30//!     "//! assert_eq!(majordomus_cli::two(), 2);\n",
31//!     "//! ```\n",
32//!     "/// Answers with two, every time, which is more than the signature promises.\n",
33//!     "///\n",
34//!     "/// ```\n",
35//!     "/// assert_eq!(majordomus_cli::two(), 2);\n",
36//!     "/// ```\n",
37//!     "pub fn two() -> usize { 2 }\n",
38//!     "#[cfg(test)]\n",
39//!     "mod tests { #[test] fn it_is_two() { assert_eq!(super::two(), 2); } }\n",
40//! )).unwrap();
41//!
42//! let report = report::rust_only(dir.path(), "a-crate").unwrap();
43//! assert!(report.passes(), "{:?}", report.violations);
44//! assert_eq!(report.modules.modules, 1);
45//! assert_eq!(report.public_api.owe_example, 1);
46//! ```
47
48use std::path::Path;
49
50use crate::capability::CapabilityRegistry;
51use crate::cli::CommandDoc;
52use crate::error::Error;
53
54use super::model::{
55    Exemption, ModuleQuality, PublicApiQuality, QualityReport, Violation, ViolationCode, SCHEMA,
56};
57use super::policy::{ExampleRequirement, Verdict, MIN_MODULE_PROSE_WORDS, MIN_PROSE_WORDS};
58use super::source::{Inventory, ItemKind};
59
60/// Measure the Rust surface of a crate: documentation, examples and module coverage.
61///
62/// The operations half needs a registry and a command tree; a caller that has neither —
63/// a test over a fixture crate, for instance — uses this and gets a report whose
64/// `operations` section is all zeroes rather than a wrong one.
65///
66/// ```
67/// use majordomus_cli::quality::report;
68/// let dir = tempfile::tempdir().unwrap();
69/// let src = dir.path().join("src");
70/// std::fs::create_dir_all(&src).unwrap();
71/// std::fs::write(src.join("lib.rs"), "//! Thin.\npub fn go() {}\n").unwrap();
72///
73/// let r = report::rust_only(dir.path(), "thin").unwrap();
74/// assert!(!r.passes(), "a thin header and an unexampled function are both findings");
75/// assert_eq!(r.operations.canonical, 0, "nothing was measured about operations");
76/// ```
77pub fn rust_only(crate_dir: &Path, target: &str) -> Result<QualityReport, Error> {
78    let inventory = Inventory::of_crate(crate_dir)?;
79    Ok(assemble(&inventory, target, None))
80}
81
82/// Measure everything: the Rust surface and the transport parity of the operations.
83///
84/// `crate_dir` is the directory holding `Cargo.toml`; `target` is what the report calls
85/// what it measured, repository-relative. The OpenAPI document is passed in rather than
86/// built here, because the caller already has one and building a second would be the
87/// duplication this subsystem exists to find.
88pub fn inspect(
89    crate_dir: &Path,
90    target: &str,
91    registry: &CapabilityRegistry,
92    tree: &CommandDoc,
93    openapi: &serde_json::Value,
94) -> Result<QualityReport, Error> {
95    let inventory = Inventory::of_crate(crate_dir)?;
96    Ok(assemble(
97        &inventory,
98        target,
99        Some((registry, tree, openapi)),
100    ))
101}
102
103fn assemble(
104    inventory: &Inventory,
105    target: &str,
106    operations: Option<(&CapabilityRegistry, &CommandDoc, &serde_json::Value)>,
107) -> QualityReport {
108    let mut report = QualityReport {
109        schema: SCHEMA.to_string(),
110        target: target.to_string(),
111        ..QualityReport::default()
112    };
113    let (api, module_quality, mut violations) = rust_surface(inventory, target);
114    report.public_api = api;
115    report.modules = module_quality;
116
117    if let Some((registry, tree, openapi)) = operations {
118        let (counts, findings) = super::parity::inspect(registry, tree, openapi);
119        report.operations = counts;
120        violations.extend(findings);
121    }
122
123    report.violations = violations;
124    report.sort();
125    report
126}
127
128/// The item and module halves of the measurement.
129fn rust_surface(
130    inventory: &Inventory,
131    target: &str,
132) -> (PublicApiQuality, ModuleQuality, Vec<Violation>) {
133    let mut api = PublicApiQuality::default();
134    let mut modules = ModuleQuality::default();
135    let mut violations = Vec::new();
136    let mut exempt: std::collections::BTreeMap<&'static str, usize> = Default::default();
137
138    for item in inventory.exported() {
139        let at = |file: &str| format!("{target}/{file}");
140        api.items += 1;
141        let documented = !item.doc.trim().is_empty();
142        if documented {
143            api.documented += 1;
144        }
145
146        if item.kind == ItemKind::Module {
147            modules.modules += 1;
148            measure_module(item, inventory, target, &mut modules, &mut violations);
149            continue;
150        }
151
152        // ---- documentation: the compiler already refuses the absence, so what is left is
153        // the ceremonial form of it. The length bar applies where a signature can hide
154        // something — a type, a function, a method, a macro — and not to a field, a
155        // variant or a constant, which are named and typed and whose gloss is often
156        // complete in four words. Demanding more of those buys padding, which is the same
157        // trade the example policy refuses to make.
158        if !documented {
159            violations.push(Violation::new(
160                ViolationCode::RustPublicMissingDocs,
161                &item.path,
162                at(&item.file),
163                Some(item.line),
164                format!(
165                    "this exported {} carries no documentation",
166                    item.kind.noun()
167                ),
168            ));
169        } else if item.kind.carries_behaviour() && item.prose_words() < MIN_PROSE_WORDS {
170            violations.push(Violation::new(
171                ViolationCode::RustPublicThinDocs,
172                &item.path,
173                at(&item.file),
174                Some(item.line),
175                format!(
176                    "{} words of prose: too few to say anything the signature has not",
177                    item.prose_words()
178                ),
179            ));
180        }
181
182        // ---- examples
183        let requirement = ExampleRequirement::of(item);
184        let Some(reason) = requirement.reason() else {
185            api.owe_example += 1;
186            match Verdict::best(&item.executable_examples_or_all(), &item.name) {
187                Some(Verdict::Counts) => api.exampled += 1,
188                Some(verdict) => {
189                    let code = verdict.code().expect("a non-counting verdict has a code");
190                    violations.push(Violation::new(
191                        code,
192                        &item.path,
193                        at(&item.file),
194                        Some(item.line),
195                        example_message(verdict, &item.name),
196                    ));
197                }
198                None => violations.push(Violation::new(
199                    ViolationCode::RustPublicMissingExample,
200                    &item.path,
201                    at(&item.file),
202                    Some(item.line),
203                    format!(
204                        "this exported {} carries behaviour and no example of it",
205                        item.kind.noun()
206                    ),
207                )),
208            }
209            continue;
210        };
211        *exempt.entry(reason).or_default() += 1;
212    }
213
214    api.exempt = exempt
215        .into_iter()
216        .map(|(reason, items)| Exemption {
217            reason: reason.to_string(),
218            items,
219        })
220        .collect();
221    (api, modules, violations)
222}
223
224fn measure_module(
225    item: &super::source::Item,
226    inventory: &Inventory,
227    target: &str,
228    modules: &mut ModuleQuality,
229    violations: &mut Vec<Violation>,
230) {
231    let at = format!("{target}/{}", item.file);
232    let documented = !item.doc.trim().is_empty();
233    if documented {
234        modules.documented += 1;
235    }
236    if !documented {
237        violations.push(Violation::new(
238            ViolationCode::RustModuleMissingDocs,
239            &item.path,
240            &at,
241            Some(item.line),
242            "this exported module carries no //! header".to_string(),
243        ));
244    } else if item.prose_words() < MIN_MODULE_PROSE_WORDS {
245        violations.push(Violation::new(
246            ViolationCode::RustModuleMissingDocs,
247            &item.path,
248            &at,
249            Some(item.line),
250            format!(
251                "{} words of header: a module is a boundary and this does not explain one",
252                item.prose_words()
253            ),
254        ));
255    }
256
257    // a module's example is judged against the module's own name, and against the last
258    // segment of any item it holds: a lifecycle example names what it exercises
259    let subject = &item.name;
260    match Verdict::best(&item.executable_examples_or_all(), subject) {
261        Some(Verdict::Counts) => modules.exampled += 1,
262        Some(Verdict::DoesNotNameSubject) => {
263            // a module header may show a lifecycle through the items it holds rather than
264            // through its own name, which is the normal and better shape
265            if item
266                .executable_examples()
267                .iter()
268                .any(|e| names_something_of(&e.code(), inventory, &item.path))
269            {
270                modules.exampled += 1;
271            } else {
272                violations.push(Violation::new(
273                    ViolationCode::RustModuleMissingExample,
274                    &item.path,
275                    &at,
276                    Some(item.line),
277                    "the header's example exercises nothing this module declares".to_string(),
278                ));
279            }
280        }
281        Some(verdict) => violations.push(Violation::new(
282            verdict.code().expect("a non-counting verdict has a code"),
283            &item.path,
284            &at,
285            Some(item.line),
286            example_message(verdict, subject),
287        )),
288        None => violations.push(Violation::new(
289            ViolationCode::RustModuleMissingExample,
290            &item.path,
291            &at,
292            Some(item.line),
293            "the //! header carries no executable example of the module's lifecycle".to_string(),
294        )),
295    }
296
297    if inventory.exercised_by_a_test(&item.path) {
298        modules.behaviourally_tested += 1;
299    } else {
300        violations.push(Violation::new(
301            ViolationCode::RustModuleMissingBehaviouralTest,
302            &item.path,
303            &at,
304            Some(item.line),
305            "no #[cfg(test)] test stands beside it and no test under tests/ names it".to_string(),
306        ));
307    }
308}
309
310/// Does the example exercise something this module declares?
311fn names_something_of(code: &str, inventory: &Inventory, module: &str) -> bool {
312    let prefix = format!("{module}::");
313    inventory
314        .items
315        .iter()
316        .filter(|i| i.exported && i.owner == module && i.path.starts_with(&prefix))
317        .any(|i| super::policy::names(code, &i.name))
318}
319
320fn example_message(verdict: Verdict, subject: &str) -> String {
321    match verdict {
322        Verdict::Counts => String::new(),
323        Verdict::Placeholder => {
324            "every example asserts only what is true of any program".to_string()
325        }
326        Verdict::DoesNotNameSubject => {
327            format!("no example names `{subject}`, so none is evidence about it")
328        }
329        Verdict::NotExecutable => {
330            "every fenced block is ignored or is prose, so the toolchain compiles none of them"
331                .to_string()
332        }
333    }
334}
335
336impl super::source::Item {
337    /// The blocks to judge: the executable ones when there are any, and otherwise every
338    /// block, so that an item whose only block is `ignore`d is told that rather than being
339    /// told it has no example.
340    fn executable_examples_or_all(&self) -> Vec<&super::source::Example> {
341        let executable = self.executable_examples();
342        if executable.is_empty() {
343            self.examples.iter().collect()
344        } else {
345            executable
346        }
347    }
348}
349
350#[cfg(test)]
351mod tests {
352    use super::*;
353
354    fn crate_of(lib: &str) -> tempfile::TempDir {
355        let dir = tempfile::tempdir().unwrap();
356        let src = dir.path().join("src");
357        std::fs::create_dir_all(&src).unwrap();
358        std::fs::write(src.join("lib.rs"), lib).unwrap();
359        dir
360    }
361
362    const HEADER: &str = "//! A fixture crate whose header is long enough to be a header: it says what the\n//! module owns, which is the one function below it and nothing else at all.\n//!\n//! ```\n//! assert_eq!(majordomus_cli::two(), 2);\n//! ```\n";
363
364    #[test]
365    fn a_documented_exampled_and_tested_crate_has_no_findings() {
366        let dir = crate_of(&format!(
367            "{HEADER}/// Answers with two, which the signature does not promise.\n///\n/// ```\n/// assert_eq!(majordomus_cli::two(), 2);\n/// ```\npub fn two() -> usize {{ 2 }}\n#[cfg(test)]\nmod tests {{ #[test] fn it_is_two() {{ assert_eq!(super::two(), 2); }} }}\n"
368        ));
369        let r = rust_only(dir.path(), "fixture").unwrap();
370        assert!(r.passes(), "{:#?}", r.violations);
371        assert_eq!(r.public_api.owe_example, 1);
372        assert_eq!(r.public_api.exampled, 1);
373        assert_eq!(r.modules.behaviourally_tested, 1);
374    }
375
376    #[test]
377    fn each_way_of_failing_raises_its_own_code_and_nothing_else() {
378        // no example at all
379        let dir = crate_of(&format!(
380            "{HEADER}/// Answers with two, which the signature does not promise.\npub fn two() -> usize {{ 2 }}\n#[cfg(test)]\nmod tests {{ #[test] fn t() {{ assert_eq!(super::two(), 2); }} }}\n"
381        ));
382        let r = rust_only(dir.path(), "f").unwrap();
383        assert_eq!(r.of(ViolationCode::RustPublicMissingExample).len(), 1);
384        assert_eq!(r.violations.len(), 1, "{:#?}", r.violations);
385
386        // an ignored one
387        let dir = crate_of(&format!(
388            "{HEADER}/// Answers with two, which the signature does not promise.\n///\n/// ```ignore\n/// two();\n/// ```\npub fn two() -> usize {{ 2 }}\n#[cfg(test)]\nmod tests {{ #[test] fn t() {{ assert_eq!(super::two(), 2); }} }}\n"
389        ));
390        let r = rust_only(dir.path(), "f").unwrap();
391        assert_eq!(r.of(ViolationCode::RustExampleNotExecutable).len(), 1);
392
393        // a placeholder
394        let dir = crate_of(&format!(
395            "{HEADER}/// Answers with two, which the signature does not promise.\n///\n/// ```\n/// let _ = two;\n/// assert!(true);\n/// ```\npub fn two() -> usize {{ 2 }}\n#[cfg(test)]\nmod tests {{ #[test] fn t() {{ assert_eq!(super::two(), 2); }} }}\n"
396        ));
397        let r = rust_only(dir.path(), "f").unwrap();
398        assert_eq!(r.of(ViolationCode::RustExamplePlaceholder).len(), 1);
399
400        // an example about something else entirely
401        let dir = crate_of(&format!(
402            "{HEADER}/// Answers with two, which the signature does not promise.\n///\n/// ```\n/// assert_eq!(1 + 1, 2, \"{{}}\", 2);\n/// let v: Vec<u8> = Vec::new();\n/// assert!(v.is_empty());\n/// ```\npub fn two() -> usize {{ 2 }}\n#[cfg(test)]\nmod tests {{ #[test] fn t() {{ assert_eq!(super::two(), 2); }} }}\n"
403        ));
404        let r = rust_only(dir.path(), "f").unwrap();
405        assert_eq!(
406            r.of(ViolationCode::RustExampleDoesNotNameSubject).len(),
407            1,
408            "{:#?}",
409            r.violations
410        );
411    }
412
413    #[test]
414    fn a_module_with_no_test_beside_it_and_none_naming_it_is_reported() {
415        let dir = crate_of(&format!(
416            "{HEADER}/// Answers with two, which the signature does not promise.\n///\n/// ```\n/// assert_eq!(majordomus_cli::two(), 2);\n/// ```\npub fn two() -> usize {{ 2 }}\n"
417        ));
418        let r = rust_only(dir.path(), "f").unwrap();
419        assert_eq!(
420            r.of(ViolationCode::RustModuleMissingBehaviouralTest).len(),
421            1
422        );
423        assert_eq!(r.modules.behaviourally_tested, 0);
424    }
425
426    #[test]
427    fn what_the_policy_does_not_ask_for_is_counted_and_named() {
428        let dir = crate_of(&format!(
429            "{HEADER}/// A limit that exists for a reason worth writing down here.\npub const LIMIT: usize = 3;\n#[cfg(test)]\nmod tests {{ #[test] fn t() {{ assert_eq!(super::LIMIT, 3); }} }}\n"
430        ));
431        let r = rust_only(dir.path(), "f").unwrap();
432        assert_eq!(r.public_api.owe_example, 0);
433        assert_eq!(r.public_api.exempt.len(), 1);
434        assert_eq!(r.public_api.exempt[0].items, 1);
435        assert!(r.public_api.exempt[0].reason.contains("value"));
436        assert!(r.passes(), "{:#?}", r.violations);
437    }
438
439    #[test]
440    fn a_report_of_the_same_tree_twice_is_the_same_report() {
441        let dir = crate_of(&format!("{HEADER}pub fn two() -> usize {{ 2 }}\n"));
442        let a = rust_only(dir.path(), "f").unwrap();
443        let b = rust_only(dir.path(), "f").unwrap();
444        assert_eq!(a, b);
445    }
446}
majordomus_cli/quality/report.rs

majordomus_cli/quality/
report.rs
