majordomus_cli/web/report/
tests.rs

1//! The test report: the suite's own results, rendered.
2//!
3//! The evidence is what the runs already write — `test/run.sh`'s TSV report for the
4//! behavioural cases and `cargo test`'s summary lines for the crate — and this module
5//! parses, joins and renders it. It decides nothing about what passed: a case that failed
6//! is a case the runner said failed, and a report that disagreed with the runner would be
7//! worse than no report.
8
9use std::path::Path;
10
11use serde::{Deserialize, Serialize};
12
13use super::{html, Origin};
14use crate::error::{Error, Result};
15
16/// The identity of the surface this report owns.
17///
18/// Named here rather than at each call site because the UI conformance report is a section
19/// of this same surface (`/tests/ui`) and has to name the surface it writes into. Two
20/// spellings of one identity is how a directory ends up declared twice, differently.
21pub const SURFACE_ID: &str = "tests";
22/// The surface's title, in a listing.
23pub const SURFACE_TITLE: &str = "Test results";
24/// What produces it, for a reader who has to rebuild it.
25pub const SURFACE_PRODUCER: &str =
26    "bash test/run.sh + cargo test, rendered by majordomus web report tests";
27
28/// One behavioural case, as the runner recorded it.
29#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
30pub struct Case {
31    /// The case's name, which is its file's stem.
32    pub name: String,
33    /// `ok` or `FAIL`, as the runner wrote it.
34    pub result: String,
35    /// How long it took, in whole seconds.
36    pub seconds: u64,
37    /// The phase it ran in: parallel, exclusive or serial.
38    pub phase: String,
39}
40
41impl Case {
42    /// Did it pass?
43    pub fn passed(&self) -> bool {
44        self.result.eq_ignore_ascii_case("ok")
45    }
46}
47
48/// A whole run: the cases, and what the crate's own tests said.
49#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
50pub struct Run {
51    /// The behavioural cases, in the order the report recorded them.
52    pub cases: Vec<Case>,
53    /// The crate's test totals, when a cargo run was given.
54    #[serde(skip_serializing_if = "Option::is_none")]
55    pub crate_tests: Option<CrateTests>,
56}
57
58/// What `cargo test` reported, summed over its binaries.
59#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
60pub struct CrateTests {
61    /// Tests that passed.
62    pub passed: usize,
63    /// Tests that failed.
64    pub failed: usize,
65    /// Tests that were ignored.
66    pub ignored: usize,
67}
68
69impl Run {
70    /// How many cases passed.
71    pub fn passed(&self) -> usize {
72        self.cases.iter().filter(|c| c.passed()).count()
73    }
74
75    /// How many cases failed.
76    pub fn failed(&self) -> usize {
77        self.cases.len() - self.passed()
78    }
79
80    /// The whole run's wall time, as the sum of its cases.
81    pub fn seconds(&self) -> u64 {
82        self.cases.iter().map(|c| c.seconds).sum()
83    }
84
85    /// Did everything the run measured pass?
86    pub fn green(&self) -> bool {
87        self.failed() == 0 && self.crate_tests.map(|t| t.failed == 0).unwrap_or(true)
88    }
89}
90
91/// Parse the runner's TSV: `name <TAB> result <TAB> seconds <TAB> phase`, one case per line.
92///
93/// A malformed line is refused rather than skipped: a report that silently dropped a case
94/// would claim a run was smaller than it was.
95///
96/// ```
97/// use majordomus_cli::web::report::tests::parse_cases;
98/// let run = parse_cases("69_context\tok\t12\tparallel\n99_adr\tFAIL\t3\texclusive\n").unwrap();
99/// assert_eq!(run.passed(), 1);
100/// assert_eq!(run.failed(), 1);
101/// assert_eq!(run.seconds(), 15);
102/// ```
103pub fn parse_cases(text: &str) -> Result<Run> {
104    let mut cases = Vec::new();
105    for (n, line) in text.lines().enumerate() {
106        if line.trim().is_empty() {
107            continue;
108        }
109        let fields: Vec<&str> = line.split('\t').collect();
110        if fields.len() < 4 {
111            return Err(Error::InvalidSurface {
112                surface: "tests".into(),
113                reason: format!(
114                    "line {} of the run report has {} field(s); it is name<TAB>result<TAB>seconds<TAB>phase",
115                    n + 1,
116                    fields.len()
117                ),
118            });
119        }
120        cases.push(Case {
121            name: fields[0].to_string(),
122            result: fields[1].to_string(),
123            seconds: fields[2].parse().unwrap_or(0),
124            phase: fields[3].to_string(),
125        });
126    }
127    Ok(Run {
128        cases,
129        crate_tests: None,
130    })
131}
132
133/// Read `cargo test`'s human output and sum its result lines.
134///
135/// The machine format is still unstable, so this reads what every cargo prints:
136/// `test result: ok. 93 passed; 0 failed; 0 ignored; ...`, once per test binary.
137///
138/// ```
139/// use majordomus_cli::web::report::tests::parse_crate_tests;
140/// let t = parse_crate_tests("test result: ok. 93 passed; 0 failed; 2 ignored; 0 measured\n\
141///                            test result: FAILED. 5 passed; 1 failed; 0 ignored; 0 measured\n");
142/// assert_eq!(t.passed, 98);
143/// assert_eq!(t.failed, 1);
144/// assert_eq!(t.ignored, 2);
145/// ```
146pub fn parse_crate_tests(text: &str) -> CrateTests {
147    let mut totals = CrateTests::default();
148    for line in text.lines().filter(|l| l.contains("test result:")) {
149        // the shape is "<n> passed; <n> failed; <n> ignored": read each label with the
150        // number that precedes it, rather than assuming where in the line either sits
151        let words: Vec<&str> = line.split_whitespace().collect();
152        for (i, word) in words.iter().enumerate().skip(1) {
153            let label = word.trim_end_matches(|c: char| !c.is_ascii_alphabetic());
154            let field = match label {
155                "passed" => &mut totals.passed,
156                "failed" => &mut totals.failed,
157                "ignored" => &mut totals.ignored,
158                _ => continue,
159            };
160            if let Ok(count) = words[i - 1].parse::<usize>() {
161                *field += count;
162            }
163        }
164    }
165    totals
166}
167
168/// Render the run into its own directory under the generated web root, and declare it.
169///
170/// Returns the directory written.
171pub fn render(root: &Path, run: &Run) -> Result<std::path::PathBuf> {
172    let dir = super::declare(root, SURFACE_ID, SURFACE_TITLE, SURFACE_PRODUCER)?;
173    let origin = Origin::read(root);
174    super::write_json(&dir, "results.json", run)?;
175
176    let mut rows: Vec<Vec<String>> = Vec::new();
177    let mut ordered = run.cases.clone();
178    ordered.sort_by(|a, b| {
179        b.passed()
180            .cmp(&a.passed())
181            .reverse()
182            .then(a.name.cmp(&b.name))
183    });
184    for case in &ordered {
185        let mark = if case.passed() {
186            "<span class=\"pass\">ok</span>".to_string()
187        } else {
188            format!("<span class=\"fail\">{}</span>", html::escape(&case.result))
189        };
190        rows.push(vec![
191            format!("<span class=\"mono\">{}</span>", html::escape(&case.name)),
192            mark,
193            format!("<span class=\"num\">{}</span>", case.seconds),
194            html::escape(&case.phase),
195        ]);
196    }
197
198    let mut summary_items = vec![
199        ("cases", run.cases.len().to_string()),
200        ("passed", run.passed().to_string()),
201        ("failed", run.failed().to_string()),
202        ("seconds", run.seconds().to_string()),
203    ];
204    if let Some(crate_tests) = run.crate_tests {
205        summary_items.push(("crate tests", crate_tests.passed.to_string()));
206        summary_items.push(("crate failures", crate_tests.failed.to_string()));
207    }
208
209    let verdict = if run.green() {
210        "Every case the run measured passed."
211    } else {
212        "The run has failures; each is named below."
213    };
214    // the UI conformance audit writes its own section of this surface; link it when it is
215    // there, and say nothing when it is not, rather than linking a page that answers 404
216    let ui = if dir.join(super::ui::SECTION).join("index.html").is_file() {
217        format!(
218            "<h2>UI conformance</h2><p><a href=\"{}/\">What a browser found on every page of \
219             the built site</a>.</p>",
220            super::ui::SECTION
221        )
222    } else {
223        String::new()
224    };
225    let body = format!(
226        "{}<h2>Behavioural cases</h2>{}{}{}",
227        html::summary(&summary_items),
228        html::table(&["case", "result", "seconds", "phase"], &rows),
229        ui,
230        html::origin(&origin, &[("results.json", "results.json")])
231    );
232    let page = html::page("Test results", verdict, &body);
233    super::write(&dir, "index.html", &page)?;
234    Ok(dir)
235}
236
237#[cfg(test)]
238mod unit {
239    use super::*;
240
241    #[test]
242    fn a_malformed_line_is_refused_rather_than_dropped() {
243        let err = parse_cases("only-a-name\n").unwrap_err().to_string();
244        assert!(err.contains("field"), "{err}");
245    }
246
247    #[test]
248    fn a_rendered_report_declares_itself_and_carries_its_evidence() {
249        let tmp = tempfile::tempdir().unwrap();
250        let run = parse_cases("a\tok\t1\tparallel\nb\tFAIL\t2\texclusive\n").unwrap();
251        let dir = render(tmp.path(), &run).unwrap();
252        let page = std::fs::read_to_string(dir.join("index.html")).unwrap();
253        assert!(page.contains("Test results"));
254        assert!(page.contains("failures"), "a red run says so: {page}");
255        assert!(dir.join("results.json").is_file());
256        let surfaces = crate::web::discover::generated(tmp.path()).unwrap();
257        assert_eq!(surfaces.len(), 1);
258        assert_eq!(surfaces[0].mount.as_str(), "/tests");
259    }
260
261    #[test]
262    fn a_green_run_says_so() {
263        let tmp = tempfile::tempdir().unwrap();
264        let run = parse_cases("a\tok\t1\tparallel\n").unwrap();
265        let dir = render(tmp.path(), &run).unwrap();
266        let page = std::fs::read_to_string(dir.join("index.html")).unwrap();
267        assert!(
268            page.contains("Every case the run measured passed"),
269            "{page}"
270        );
271    }
272}
majordomus_cli/web/report/tests.rs

majordomus_cli/web/report/
tests.rs
