majordomus_cli/web/report/
ui.rs

1//! The UI conformance report: what a browser found on every page of the built site.
2//!
3//! The evidence is the results document `scripts/ui audit` writes — the pages it visited,
4//! the widths it visited them at, and every finding, each already naming the rule it broke
5//! and the elements that broke it. This module parses and renders it. It decides nothing:
6//! a page is conformant here exactly when the audit found nothing on it, and a report that
7//! disagreed with the audit would be worse than no report.
8//!
9//! The rendering is a *section of the test surface* rather than a surface of its own. A UI
10//! conformance run is a test run, `/tests/ui` is where a reader looks for it, and the
11//! topology refuses a surface mounted inside another's subtree (`surface.nested-mount`), so
12//! the architecture has already answered where this belongs.
13
14use std::collections::BTreeMap;
15use std::path::{Path, PathBuf};
16
17use serde::{Deserialize, Serialize};
18use serde_json::Value;
19
20use super::{html, tests, Origin};
21use crate::error::{Error, Result};
22
23/// The contract of the results document, written by `scripts/lib/ui-run.mjs`.
24pub const RESULTS_SCHEMA: &str = "ui-audit/v1";
25
26/// The directory of this report inside the test surface, and the path it is read at.
27pub const SECTION: &str = "ui";
28
29/// One element a finding names, as the browser saw it.
30///
31/// The selector is the only field every rule produces; a rule that measured more — how far
32/// past the viewport an element reached, what its position was — carries it here rather
33/// than in prose, so the report can render it without knowing which rule it came from.
34#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
35pub struct Element {
36    /// A selector a person can paste into the browser's console.
37    pub selector: String,
38    /// Whatever else the rule measured about it.
39    #[serde(flatten)]
40    pub measured: BTreeMap<String, Value>,
41}
42
43/// One thing wrong on one page at one width.
44#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
45pub struct Finding {
46    /// The page.
47    pub route: String,
48    /// The width it was visited at, in CSS pixels.
49    pub width: u32,
50    /// How thoroughly the page was visited: `sweep` or `critical`.
51    #[serde(default)]
52    pub tier: String,
53    /// The rule it broke, stable enough to grep for.
54    pub rule: String,
55    /// What is wrong, in one line.
56    pub detail: String,
57    /// The elements that broke it, when the rule could name them.
58    #[serde(default)]
59    pub elements: Vec<Element>,
60}
61
62/// One surface the audit measured, and what it contributed.
63///
64/// A built surface is a directory, and every page of it is visited. A surface the executable
65/// renders has no directory: its routes are crawled out of its own anchors and a sample of
66/// each family is visited, so the number of routes it *has* is reported beside the number
67/// that was *sampled*. A sample rendered as a total would be a report overstating its own
68/// coverage, and the Cockpit's thousands of routes are why that distinction belongs on the
69/// page rather than in somebody's head.
70///
71/// ```
72/// use majordomus_cli::web::report::ui::{parse, Surface};
73/// let run = parse(r#"{"schema":"ui-audit/v1","pages":108,"visits":478,"findings":[],
74///     "surfaces":[{"id":"cockpit","mount":"/cockpit","kind":"native-route",
75///                  "routes":2660,"families":14,"sampled":108}]}"#).unwrap();
76/// let surface: &Surface = &run.surfaces[0];
77/// assert_eq!((surface.routes, surface.sampled), (2660, 108));
78/// assert!(!surface.truncated, "a crawl that hit its budget reports a floor, not a total");
79/// ```
80#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
81pub struct Surface {
82    /// The surface's id in the topology.
83    pub id: String,
84    /// Where the executable serves it.
85    pub mount: String,
86    /// `static-directory` or `native-route`.
87    #[serde(default)]
88    pub kind: String,
89    /// How many routes the surface has, when they were crawled rather than read from disk.
90    #[serde(default)]
91    pub routes: usize,
92    /// How many families those routes fell into.
93    #[serde(default)]
94    pub families: usize,
95    /// How many of them the audit visited.
96    #[serde(default)]
97    pub sampled: usize,
98    /// Whether the crawl hit its budget, so the route count is a floor rather than a total.
99    #[serde(default)]
100    pub truncated: bool,
101}
102
103/// A subtree of a page the engine was told to skip, and what declared it.
104///
105/// A page may carry a component tree that is not this repository's to fix — pinned at a
106/// version, its markup and its stylesheet arriving together. The page declares it with
107/// `data-mj-foreign`; the engine skips it, and the report says so. "Not measured" and
108/// "measured and clean" are different claims, and a report that conflated them would be
109/// worth nothing.
110///
111/// ```
112/// use majordomus_cli::web::report::ui::{parse, Foreign};
113/// let run = parse(r#"{"schema":"ui-audit/v1","pages":1,"visits":14,"findings":[],
114///     "foreign":[{"route":"/swagger","selector":"main#swagger-ui",
115///                 "declares":"swagger-ui-dist@5.17.14"}]}"#).unwrap();
116/// let skipped: &Foreign = &run.foreign[0];
117/// assert_eq!(skipped.declares, "swagger-ui-dist@5.17.14");
118/// assert!(run.green(), "a page around a skipped subtree can still be clean");
119/// ```
120#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
121pub struct Foreign {
122    /// A page it was seen on.
123    #[serde(default)]
124    pub route: String,
125    /// The element that carried the declaration.
126    #[serde(default)]
127    pub selector: String,
128    /// What the page said it is: the third party and its pinned version.
129    #[serde(default)]
130    pub declares: String,
131}
132
133/// A whole audit run.
134#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
135pub struct Run {
136    /// The contract of this document.
137    pub schema: String,
138    /// The origin the audit drove.
139    #[serde(default)]
140    pub origin: String,
141    /// Whether every discovered page was visited, or the run was narrowed for iteration.
142    #[serde(default)]
143    pub complete: bool,
144    /// Where the pages and the widths came from.
145    #[serde(default)]
146    pub source: BTreeMap<String, String>,
147    /// The breakpoints the compiled stylesheet declared.
148    #[serde(default)]
149    pub breakpoints: Vec<u32>,
150    /// The widths the sweep visits.
151    #[serde(default)]
152    pub viewports: Vec<u32>,
153    /// Every surface the run measured, and what each contributed.
154    #[serde(default)]
155    pub surfaces: Vec<Surface>,
156    /// Every subtree the engine was told to skip, once per declaration.
157    #[serde(default)]
158    pub foreign: Vec<Foreign>,
159    /// How many pages were visited.
160    pub pages: usize,
161    /// How many page-and-width visits that came to.
162    pub visits: usize,
163    /// How long the run took, in whole seconds.
164    #[serde(default)]
165    pub seconds: u64,
166    /// Every finding, in the order the run produced them.
167    pub findings: Vec<Finding>,
168}
169
170impl Run {
171    /// Did every page pass?
172    pub fn green(&self) -> bool {
173        self.findings.is_empty()
174    }
175
176    /// The findings per rule, most frequent first, then by rule name.
177    ///
178    /// Derived rather than read from the document: a count and a list that can disagree is
179    /// a report that will one day lie about its own evidence.
180    ///
181    /// ```
182    /// use majordomus_cli::web::report::ui::parse;
183    /// let run = parse(r#"{"schema":"ui-audit/v1","pages":1,"visits":1,"findings":[
184    ///     {"route":"/","width":320,"rule":"a","detail":"x"},
185    ///     {"route":"/","width":320,"rule":"a","detail":"y"},
186    ///     {"route":"/","width":320,"rule":"b","detail":"z"}]}"#).unwrap();
187    /// assert_eq!(run.by_rule(), vec![("a".to_string(), 2), ("b".to_string(), 1)]);
188    /// ```
189    pub fn by_rule(&self) -> Vec<(String, usize)> {
190        let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
191        for finding in &self.findings {
192            *counts.entry(finding.rule.as_str()).or_default() += 1;
193        }
194        let mut ordered: Vec<(String, usize)> = counts
195            .into_iter()
196            .map(|(k, v)| (k.to_string(), v))
197            .collect();
198        ordered.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
199        ordered
200    }
201
202    /// The pages a rule was found on, deduplicated and ordered.
203    pub fn routes_of(&self, rule: &str) -> Vec<&str> {
204        let mut routes: Vec<&str> = self
205            .findings
206            .iter()
207            .filter(|f| f.rule == rule)
208            .map(|f| f.route.as_str())
209            .collect();
210        routes.sort_unstable();
211        routes.dedup();
212        routes
213    }
214}
215
216/// Read a results document, refusing a contract this executable does not know.
217pub fn parse(text: &str) -> Result<Run> {
218    let run: Run = serde_json::from_str(text).map_err(|e| Error::InvalidSurface {
219        surface: "ui".into(),
220        reason: format!(
221            "the results document does not parse: {e}; the contract is {RESULTS_SCHEMA}"
222        ),
223    })?;
224    if run.schema != RESULTS_SCHEMA {
225        return Err(Error::InvalidSurface {
226            surface: "ui".into(),
227            reason: format!(
228                "the results document declares schema '{}', and this executable reads {RESULTS_SCHEMA}",
229                run.schema
230            ),
231        });
232    }
233    Ok(run)
234}
235
236/// Read a results document from disk.
237pub fn read(path: &Path) -> Result<Run> {
238    let text =
239        std::fs::read_to_string(path).map_err(|e| Error::io(path.display().to_string(), e))?;
240    parse(&text)
241}
242
243/// Render the run into `/tests/ui`, inside the test surface.
244///
245/// Returns the directory written.
246pub fn render(root: &Path, run: &Run) -> Result<PathBuf> {
247    // the enclosing surface must exist for this section to be reachable, and belongs to the
248    // test report: declared here only when nobody has declared it, never overwritten
249    let surface = super::declare_if_absent(
250        root,
251        tests::SURFACE_ID,
252        tests::SURFACE_TITLE,
253        tests::SURFACE_PRODUCER,
254    )?;
255    let dir = surface.join(SECTION);
256    std::fs::create_dir_all(&dir).map_err(|e| Error::io(dir.display().to_string(), e))?;
257    let origin = Origin::read(root);
258    super::write_json(&dir, "results.json", run)?;
259
260    let mut summary = vec![
261        ("pages", run.pages.to_string()),
262        ("visits", run.visits.to_string()),
263        ("findings", run.findings.len().to_string()),
264        ("rules broken", run.by_rule().len().to_string()),
265        ("seconds", run.seconds.to_string()),
266    ];
267    if !run.viewports.is_empty() {
268        summary.push(("widths", run.viewports.len().to_string()));
269    }
270
271    let mut rule_rows: Vec<Vec<String>> = Vec::new();
272    for (rule, count) in run.by_rule() {
273        let routes = run.routes_of(&rule);
274        let example = routes.first().copied().unwrap_or("");
275        rule_rows.push(vec![
276            format!("<span class=\"mono\">{}</span>", html::escape(&rule)),
277            format!("<span class=\"num\">{count}</span>"),
278            format!("<span class=\"num\">{}</span>", routes.len()),
279            format!("<span class=\"mono\">{}</span>", html::escape(example)),
280        ]);
281    }
282
283    let mut finding_rows: Vec<Vec<String>> = Vec::new();
284    for finding in &run.findings {
285        let elements = finding
286            .elements
287            .iter()
288            .take(3)
289            .map(|e| html::escape(&e.selector))
290            .collect::<Vec<_>>()
291            .join("<br>");
292        finding_rows.push(vec![
293            format!(
294                "<span class=\"mono\">{}</span>",
295                html::escape(&finding.route)
296            ),
297            format!("<span class=\"num\">{}</span>", finding.width),
298            format!(
299                "<span class=\"mono\">{}</span>",
300                html::escape(&finding.rule)
301            ),
302            html::escape(&finding.detail),
303            format!("<span class=\"mono\">{elements}</span>"),
304        ]);
305    }
306
307    let verdict = if run.green() {
308        "Every page the audit visited satisfies every invariant it checks."
309    } else {
310        "The audit found conformance failures; every one is named below, with the page, the \
311         width and the element."
312    };
313    let scope = if run.complete {
314        String::new()
315    } else {
316        "<p class=\"lede\"><strong>This run was narrowed</strong> — it visited part of the \
317         site, not all of it, so it cannot stand for a clean audit.</p>"
318            .to_string()
319    };
320    let provenance = format!(
321        "<h2>Where the target set came from</h2><ul><li>pages: {}</li><li>widths: {}</li>\
322         <li>breakpoints: <span class=\"mono\">{}</span></li>\
323         <li>widths visited: <span class=\"mono\">{}</span></li></ul>\
324         <p class=\"lede\">Nothing in this run names a page or a width. Add a page to the \
325         site and it is audited; change a breakpoint in the theme and the widths follow.</p>",
326        html::escape(
327            run.source
328                .get("pages")
329                .map(String::as_str)
330                .unwrap_or("the built site")
331        ),
332        html::escape(
333            run.source
334                .get("viewports")
335                .map(String::as_str)
336                .unwrap_or("the compiled stylesheet")
337        ),
338        run.breakpoints
339            .iter()
340            .map(u32::to_string)
341            .collect::<Vec<_>>()
342            .join(", "),
343        run.viewports
344            .iter()
345            .map(u32::to_string)
346            .collect::<Vec<_>>()
347            .join(", "),
348    );
349
350    // Which surfaces were measured at all. This table is the answer to the question ADR 0036
351    // left open — whether the audit reaches the surfaces the executable renders as well as
352    // the ones it serves from a directory — so it is on the page whether the run is green or
353    // not, and it says what a crawled surface was sampled *from*.
354    let surfaces_section = if run.surfaces.is_empty() {
355        String::new()
356    } else {
357        let rows: Vec<Vec<String>> = run
358            .surfaces
359            .iter()
360            .map(|surface| {
361                let native = surface.kind == "native-route";
362                let found = if !native {
363                    "its directory and its sitemap".to_string()
364                } else if surface.routes == 0 {
365                    "no pages: the mount is not a document".to_string()
366                } else {
367                    format!(
368                        "{} route(s) in {} family(ies), crawled from its own anchors{}",
369                        surface.routes,
370                        surface.families,
371                        if surface.truncated {
372                            ", and the crawl hit its budget, so that is a floor"
373                        } else {
374                            ""
375                        }
376                    )
377                };
378                vec![
379                    format!("<span class=\"mono\">{}</span>", html::escape(&surface.id)),
380                    format!(
381                        "<span class=\"mono\">{}</span>",
382                        html::escape(&surface.mount)
383                    ),
384                    format!(
385                        "<span class=\"mono\">{}</span>",
386                        html::escape(&surface.kind)
387                    ),
388                    if native {
389                        format!("<span class=\"num\">{}</span>", surface.sampled)
390                    } else {
391                        "every page".to_string()
392                    },
393                    html::escape(&found),
394                ]
395            })
396            .collect();
397        format!(
398            "<h2>The surfaces this run measured</h2>{}",
399            html::table(&["surface", "mount", "kind", "visited", "found by"], &rows)
400        )
401    };
402
403    // What was not measured, and what declared it. On the page whether the run is green or
404    // not: a reader who cannot see this cannot tell a clean audit from a narrowed one.
405    let foreign_section = if run.foreign.is_empty() {
406        String::new()
407    } else {
408        let rows: Vec<Vec<String>> = run
409            .foreign
410            .iter()
411            .map(|subtree| {
412                vec![
413                    format!(
414                        "<span class=\"mono\">{}</span>",
415                        html::escape(&subtree.declares)
416                    ),
417                    format!(
418                        "<span class=\"mono\">{}</span>",
419                        html::escape(&subtree.selector)
420                    ),
421                    format!(
422                        "<span class=\"mono\">{}</span>",
423                        html::escape(&subtree.route)
424                    ),
425                ]
426            })
427            .collect();
428        format!(
429            "<h2>Not measured, and why</h2><p class=\"lede\">A page may declare a subtree \
430             that is a third party's to answer for. The accessibility engine skipped these; \
431             everything around them was measured as usual.</p>{}",
432            html::table(&["declared as", "element", "first seen on"], &rows)
433        )
434    };
435
436    let findings_section = if run.green() {
437        String::new()
438    } else {
439        format!(
440            "<h2>By rule</h2>{}<h2>Every finding</h2>{}",
441            html::table(&["rule", "findings", "pages", "first page"], &rule_rows),
442            html::table(
443                &["page", "width", "rule", "detail", "elements"],
444                &finding_rows
445            ),
446        )
447    };
448
449    let body = format!(
450        "{}{}{}{}{}{}<p><a href=\"../\">The test run this belongs to</a></p>{}",
451        scope,
452        html::summary(&summary),
453        surfaces_section,
454        foreign_section,
455        findings_section,
456        provenance,
457        html::origin(&origin, &[("results.json", "results.json")])
458    );
459    let page = html::page("UI conformance", verdict, &body);
460    super::write(&dir, "index.html", &page)?;
461    Ok(dir)
462}
463
464#[cfg(test)]
465mod unit {
466    use super::*;
467
468    const RUN: &str = r#"{
469        "schema": "ui-audit/v1", "origin": "http://127.0.0.1:1", "complete": true,
470        "source": {"pages": "the built site", "viewports": "the stylesheet"},
471        "breakpoints": [640, 768], "viewports": [320, 639, 640, 1440],
472        "pages": 2, "visits": 4, "seconds": 7,
473        "findings": [
474            {"route": "/", "width": 320, "tier": "sweep",
475             "rule": "responsive.horizontal-overflow", "detail": "the document is 420px wide",
476             "elements": [{"selector": "pre.code", "right": 420, "width": 400}]}
477        ]
478    }"#;
479
480    #[test]
481    fn a_document_from_another_contract_is_refused_rather_than_guessed() {
482        let err = parse(r#"{"schema":"ui-audit/v2","pages":0,"visits":0,"findings":[]}"#)
483            .unwrap_err()
484            .to_string();
485        assert!(err.contains("ui-audit/v1"), "{err}");
486    }
487
488    #[test]
489    fn what_a_rule_measured_survives_the_round_trip() {
490        let run = parse(RUN).unwrap();
491        let element = &run.findings[0].elements[0];
492        assert_eq!(element.selector, "pre.code");
493        assert_eq!(element.measured.get("right").unwrap().as_i64(), Some(420));
494    }
495
496    #[test]
497    fn the_report_is_a_section_of_the_test_surface_and_declares_no_surface_of_its_own() {
498        let tmp = tempfile::tempdir().unwrap();
499        let run = parse(RUN).unwrap();
500        let dir = render(tmp.path(), &run).unwrap();
501        assert!(dir.ends_with("tests/ui"));
502        assert!(dir.join("index.html").is_file());
503        assert!(dir.join("results.json").is_file());
504        assert!(
505            !dir.join(crate::web::discover::DECLARATION_FILE).is_file(),
506            "a section of another surface declares nothing of its own"
507        );
508        let surfaces = crate::web::discover::generated(tmp.path()).unwrap();
509        assert_eq!(surfaces.len(), 1, "one surface, not two: {surfaces:?}");
510        assert_eq!(surfaces[0].mount.as_str(), "/tests");
511    }
512
513    #[test]
514    fn rendering_the_section_never_overwrites_the_enclosing_report() {
515        let tmp = tempfile::tempdir().unwrap();
516        let suite = tests::parse_cases("a\tok\t1\tparallel\n").unwrap();
517        tests::render(tmp.path(), &suite).unwrap();
518        let before = std::fs::read_to_string(
519            super::super::directory(tmp.path(), "tests").join("surface.json"),
520        )
521        .unwrap();
522        render(tmp.path(), &parse(RUN).unwrap()).unwrap();
523        let after = std::fs::read_to_string(
524            super::super::directory(tmp.path(), "tests").join("surface.json"),
525        )
526        .unwrap();
527        assert_eq!(before, after);
528    }
529
530    #[test]
531    fn a_narrowed_run_says_so_on_the_page() {
532        let tmp = tempfile::tempdir().unwrap();
533        let mut run = parse(RUN).unwrap();
534        run.complete = false;
535        let dir = render(tmp.path(), &run).unwrap();
536        let page = std::fs::read_to_string(dir.join("index.html")).unwrap();
537        assert!(page.contains("narrowed"), "{page}");
538    }
539
540    #[test]
541    fn a_clean_run_says_so_and_lists_nothing() {
542        let tmp = tempfile::tempdir().unwrap();
543        let mut run = parse(RUN).unwrap();
544        run.findings.clear();
545        let dir = render(tmp.path(), &run).unwrap();
546        let page = std::fs::read_to_string(dir.join("index.html")).unwrap();
547        assert!(page.contains("satisfies every invariant"), "{page}");
548        assert!(!page.contains("Every finding"), "{page}");
549    }
550}
majordomus_cli/web/report/ui.rs

majordomus_cli/web/report/
ui.rs
