majordomus_cli/
redaction.rs

1//! The one rule set that makes text a run printed fit to publish.
2//!
3//! Evidence carries text nobody wrote for a reader outside the machine it ran on: the reason
4//! a case skipped, the tail of a failure, a command's captured output. Any of it can hold a
5//! credential the command echoed or a path that names the account it ran under, and once it
6//! is on a public page it cannot be taken back. [`public_text`] is the rule for that text;
7//! wiring the evidence publication to it is I1955, and until then nothing calls it.
8//!
9//! [`redact_secrets`] alone is applied today wherever this process hands back or sends text
10//! it did not write (I2161): every HTTP error body (`http::router::error_response`, which
11//! `http::router::Response::error` returns), every JSON-RPC error (`mcp::protocol::error`)
12//! and MCP tool refusal (`mcp::surface::Surface::call_from`), the request fields the server
13//! logs (`http::router::Router::foreign_origin`, `http::server::answer_http`), every field of
14//! every event and span the log writes, whoever logged it (`logging::redact`, I2708), a
15//! handover body before it is signed into the mesh journal and replicated in clear
16//! (`mesh::handover::to_body`), and the free text of a review as it is signed
17//! (`mesh::cooperation::redacted`, I2708). Two records use it to find a credential rather than to send
18//! one: `continuity::record::portability` refuses a continuity record that carries one, and
19//! `reasoning::store::redact` takes it out of a reasoning record before the record is kept.
20//! These are all of its callers; `no_answer_carries_a_credential` holds this list to the code
21//! in both directions.
22//!
23//! The credential shapes are not a second table. They are a port of `MJ_CAPTURE_SECRETS` and
24//! `MJ_CAPTURE_SECRET_ASSIGN` in `lib/capture.sh`, the table the prompt archive has redacted
25//! with from the start, applied the way `mj_capture_redact` applies it: each shape in table
26//! order over the whole text, every leftmost-longest match replaced by a marker naming the
27//! shape, and then the assignment rule, which keeps the name a value was assigned to. Two
28//! tests hold the port to the table: one reads `lib/capture.sh` and compares the shapes, and
29//! one fixture, `test/fixtures/redaction/shapes.tsv`, is run through both implementations.
30//! The crate has no regular-expression engine, so the shapes are matched by the small
31//! hand-written matcher below, which knows exactly the constructs the table uses.
32//!
33//! Two neighbours are different concerns and stay apart: `execution::redact` removes the
34//! fields a capability's input schema marks sensitive, and `generate::forbidden_in` is the
35//! last word on a published artifact, which [`public_text`] asks after it has done its work.
36//! Before asking, [`public_text`] refuses a text that still carries a secret no shape covers:
37//! one whose announcing shape fired, in this call or in an earlier redaction whose marker
38//! the text carries, and one whose armour or name is left once the shapes have run. Redacting
39//! the announcement is what would let the rest past that check.
40//!
41//! ```
42//! use majordomus_cli::redaction::redact_secrets;
43//!
44//! // assembled at run time, so that no committed file carries a credential's shape
45//! let token = format!("{}{}", "ghp_", "a".repeat(24));
46//! let out = redact_secrets(&format!("pushing with {token} failed"));
47//! assert_eq!(out.text, "pushing with [redacted:github-token] failed");
48//! assert_eq!(out.kinds, ["github-token"]);
49//! ```
50
51use std::path::Path;
52
53/// Text after [`redact_secrets`] or [`public_text`], with the shapes that fired in it.
54///
55/// `kinds` is what `mj_capture_redacted_kinds` reports for the same text: the names its
56/// `[redacted:<name>]` markers carry, each once, in the crate's canonical order, which for
57/// these lower-case hyphenated names is the byte order the shell's `LC_ALL=C sort -u` gives.
58/// They are read off the text, as the shell reads them, so a marker the input already
59/// carried counts as much as one this call wrote: text redacted once and redacted again
60/// still says what was taken out of it. A marker that names no shape of the table is not a
61/// kind here, since a kind is one of [`shape_names`]; the shell would list that name too.
62///
63/// ```
64/// use majordomus_cli::redaction::{redact_secrets, Redacted};
65///
66/// let key = format!("{}{}", "sk-ant-", "b".repeat(20));
67/// let pat = format!("{}{}", "github_pat_", "c".repeat(20));
68/// let out: Redacted = redact_secrets(&format!("{pat} then {key} then {pat}"));
69/// assert_eq!(out.kinds, ["anthropic-key", "github-pat"]);
70/// assert!(!out.text.contains("sk-ant-"));
71///
72/// // redacting again finds nothing new, and still names what the text had taken out
73/// assert_eq!(redact_secrets(&out.text).kinds, out.kinds);
74/// ```
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct Redacted {
77    /// The text with every credential replaced by `[redacted:<name>]`.
78    pub text: String,
79    /// The shapes the text's markers name, sorted and unique.
80    pub kinds: Vec<&'static str>,
81}
82
83/// The names of the credential shapes, in the order they are applied, then `assignment`.
84///
85/// The order is `MJ_CAPTURE_SECRETS`' own, and it matters: the shapes are applied one after
86/// another, so an Anthropic key is named as one before the broader OpenAI shape could claim
87/// it.
88///
89/// ```
90/// use majordomus_cli::redaction::shape_names;
91///
92/// let names = shape_names();
93/// assert_eq!(names.first(), Some(&"anthropic-key"));
94/// assert_eq!(names.last(), Some(&"assignment"));
95/// let anthropic = names.iter().position(|n| *n == "anthropic-key");
96/// let openai = names.iter().position(|n| *n == "openai-key");
97/// assert!(anthropic < openai);
98/// ```
99pub fn shape_names() -> Vec<&'static str> {
100    SHAPES
101        .iter()
102        .chain(std::iter::once(&ASSIGNMENT))
103        .map(|shape| shape.name)
104        .collect()
105}
106
107/// Every credential in `text` replaced by a marker that names its shape.
108///
109/// Byte for byte what `mj_capture_redact` makes of the same text: each shape in table order
110/// over the whole text, every non-overlapping leftmost-longest match replaced as `sed -E`
111/// with `g` replaces it, then the assignment rule, which keeps the name and the separator
112/// and replaces only the value. Text with nothing credential-shaped in it comes back
113/// unchanged, prose that merely talks about a password included.
114///
115/// ```
116/// use majordomus_cli::redaction::redact_secrets;
117///
118/// let value = "d".repeat(16);
119/// let out = redact_secrets(&format!("export API_KEY={value}"));
120/// assert_eq!(out.text, "export API_KEY=[redacted:assignment]");
121/// assert_eq!(out.kinds, ["assignment"]);
122///
123/// let prose = "the password is in the manual";
124/// assert_eq!(redact_secrets(prose).text, prose);
125/// assert!(redact_secrets(prose).kinds.is_empty());
126/// ```
127pub fn redact_secrets(text: &str) -> Redacted {
128    let mut text = text.to_string();
129    for shape in SHAPES.iter().chain(std::iter::once(&ASSIGNMENT)) {
130        if let Some(replaced) = shape.replace_all(&text) {
131            text = replaced;
132        }
133    }
134    let kinds = kinds_in(&text);
135    Redacted { text, kinds }
136}
137
138/// How every marker opens; the shape's name and `]` follow.
139const MARKER: &str = "[redacted:";
140
141/// The shapes named by the markers in `text`, each once and in canonical order: what
142/// `mj_capture_redacted_kinds` reads with `grep -oE '\[redacted:[a-z-]+\]'`, for the names of
143/// the table. No marker can hold another's opening, so every opening is a candidate.
144fn kinds_in(text: &str) -> Vec<&'static str> {
145    let names = shape_names();
146    let mut kinds: Vec<&'static str> = text
147        .match_indices(MARKER)
148        .filter_map(|(at, _)| {
149            let rest = &text[at + MARKER.len()..];
150            let len = rest
151                .find(|c: char| !(c.is_ascii_lowercase() || c == '-'))
152                .unwrap_or(rest.len());
153            rest[len..].starts_with(']').then_some(&rest[..len])
154        })
155        .filter_map(|name| names.iter().copied().find(|shape| *shape == name))
156        .collect();
157    crate::order::canonical_strings(&mut kinds);
158    kinds.dedup();
159    kinds
160}
161
162/// The repository root written as `<repo>` and the home directory as `<home>`.
163///
164/// These are `mj_uc_normalise`'s first rules, in its order: the canonical (symlinks
165/// resolved) spelling of the root, then the spelling the caller gave, then the home
166/// directory. The canonical spelling goes first because the given one can be a suffix of it
167/// (a temporary directory reached through a symlink), and the root before the home because
168/// a checkout usually lives inside one. A trailing separator is not part of a spelling, and
169/// a spelling that is empty without it, the filesystem root, replaces nothing.
170///
171/// ```
172/// use majordomus_cli::redaction::normalise_machine_paths;
173///
174/// let dir = tempfile::tempdir().unwrap();
175/// let home = dir.path().join("home");
176/// let root = home.join("checkout");
177/// std::fs::create_dir_all(&root).unwrap();
178///
179/// let text = format!("{}/src/lib.rs and {}/.cache", root.display(), home.display());
180/// let out = normalise_machine_paths(&text, &root, Some(&home));
181/// assert_eq!(out, "<repo>/src/lib.rs and <home>/.cache");
182/// ```
183pub fn normalise_machine_paths(text: &str, root: &Path, home: Option<&Path>) -> String {
184    let canonical = std::fs::canonicalize(root).ok();
185    let mut out = text.to_string();
186    for spelling in [canonical.as_deref(), Some(root)]
187        .into_iter()
188        .flatten()
189        .filter_map(spelling_of)
190    {
191        out = out.replace(&spelling, "<repo>");
192    }
193    if let Some(home) = home.and_then(spelling_of) {
194        out = out.replace(&home, "<home>");
195    }
196    out
197}
198
199/// Text fit to publish: machine paths normalised, then credentials redacted, then checked.
200///
201/// A redaction covers what its shape matched, and two shapes match less than the secret
202/// they announce. So the text is refused, with what it carries and never the text, when:
203///
204/// - its markers name a private key's header, whose body follows on the lines after it, or
205///   an access key id, whose secret access key no shape matches. The markers are read off
206///   the text, so a text redacted once already is refused as the original would be.
207///   Redacting the announcement removes the very marker `generate::forbidden_in` refuses
208///   on, and would publish the rest;
209/// - it still holds a private key's armour line, such as the footer a failure's tail keeps
210///   after cutting the header off, or a secret access key assigned a value: a secret whose
211///   announcement no longer is in the text, or never was;
212/// - `generate::forbidden_in`, the check every published artifact already passes, still
213///   finds a marker: something neither rule can make safe, such as another account's home
214///   directory or a bearer header too short to be a credential's shape.
215///
216/// ```
217/// use majordomus_cli::redaction::public_text;
218///
219/// let dir = tempfile::tempdir().unwrap();
220/// let key = format!("{}{}", "xoxb-", "e".repeat(12));
221/// let text = format!("{}/target: {key}", dir.path().display());
222/// let out = public_text(&text, dir.path(), None).unwrap();
223/// assert_eq!(out.text, "<repo>/target: [redacted:slack-token]");
224///
225/// let elsewhere = format!("{}{}", "/ho", "me/someone/.cache");
226/// let refused = public_text(&elsewhere, dir.path(), None).unwrap_err();
227/// assert!(refused.contains("an absolute path on the machine"));
228///
229/// let pem = format!("{}{}\n{}", "-----BEGIN ", "PRIVATE KEY-----", "f".repeat(64));
230/// let refused = public_text(&pem, dir.path(), None).unwrap_err();
231/// assert!(refused.contains("`private-key-header`"));
232///
233/// let tail = format!("{}\n{}{}", "g".repeat(64), "-----END ", "PRIVATE KEY-----");
234/// let refused = public_text(&tail, dir.path(), None).unwrap_err();
235/// assert!(refused.contains("`PRIVATE KEY-----`"));
236/// ```
237pub fn public_text(text: &str, root: &Path, home: Option<&Path>) -> Result<Redacted, String> {
238    let redacted = redact_secrets(&normalise_machine_paths(text, root, home));
239    if let Some((marker, what)) = uncovered_secret(&redacted) {
240        return Err(format!(
241            "the text carries {what} (`{marker}`), which redacting the shapes of the table \
242             does not cover, so it is not published"
243        ));
244    }
245    match crate::generate::forbidden_in(&redacted.text) {
246        None => Ok(redacted),
247        Some((marker, what)) => Err(format!(
248            "the text still carries {what} (`{marker}`) once its machine paths are normalised \
249             and its credentials redacted, so it is not published"
250        )),
251    }
252}
253
254/// The first secret `redacted` still carries that no shape covers, as the name it is
255/// refused under and what it is.
256fn uncovered_secret(redacted: &Redacted) -> Option<(&'static str, &'static str)> {
257    let announced = ANNOUNCES_MORE
258        .iter()
259        .find(|(kind, _)| redacted.kinds.contains(kind));
260    let armour = ARMOUR
261        .iter()
262        .find(|(line, _)| contains_ignoring_case(&redacted.text, line));
263    announced.or(armour).copied().or_else(|| {
264        SECRET_ACCESS_KEY.found_in(&redacted.text).then_some((
265            SECRET_ACCESS_KEY.name,
266            "a secret access key assigned a value, which no shape of the table matches",
267        ))
268    })
269}
270
271/// The shapes whose match announces a secret it does not cover, with what that secret is.
272const ANNOUNCES_MORE: &[(&str, &str)] = &[
273    (
274        "aws-access-key-id",
275        "an access key id, whose secret access key no shape matches",
276    ),
277    (
278        "private-key-header",
279        "a private key's header, whose body no shape matches",
280    ),
281];
282
283/// The ends of a private key's armour lines, in any case: every header the shape misses
284/// and every footer, which a tail keeps when it has cut the header off.
285const ARMOUR: &[(&str, &str)] = &[
286    (
287        "PRIVATE KEY-----",
288        "a private key's armour line, whose body no shape matches",
289    ),
290    (
291        "PRIVATE KEY BLOCK-----",
292        "a PGP private key's armour line, whose body no shape matches",
293    ),
294];
295
296/// Does `text` contain `needle`, ASCII letters compared without their case?
297fn contains_ignoring_case(text: &str, needle: &str) -> bool {
298    text.as_bytes()
299        .windows(needle.len())
300        .any(|window| window.eq_ignore_ascii_case(needle.as_bytes()))
301}
302
303/// A secret access key assigned a long opaque value, in the assignment rule's own form:
304/// `aws_secret_access_key = ...`, `AWS_SECRET_ACCESS_KEY=...`, `"SecretAccessKey": "..."`.
305/// It is not a shape of the table and redacts nothing: [`public_text`] refuses a text it is
306/// found in, since the id it pairs with may be on a line the text no longer has. A name with
307/// no value after it, as a skip reason says it, is not a match.
308const SECRET_ACCESS_KEY: Shape = Shape {
309    name: "secret-access-key",
310    kept: &[
311        Atom::Caseless("secret"),
312        run("-_", 0, Some(1)),
313        Atom::Caseless("access"),
314        run("-_", 0, Some(1)),
315        Atom::Caseless("key"),
316        ASSIGNED,
317    ],
318    secret: &[OPAQUE_VALUE],
319};
320
321/// A path as the text it appears in, without a trailing separator; `None` when nothing is
322/// left, so that the filesystem root never replaces every separator in the text.
323fn spelling_of(path: &Path) -> Option<String> {
324    let lossy = path.to_string_lossy();
325    let trimmed = lossy.trim_end_matches('/');
326    (!trimmed.is_empty()).then(|| trimmed.to_string())
327}
328
329// ---------------------------------------------------------------- the table
330
331/// The inside of a POSIX bracket expression, as `lib/capture.sh` spells it: single bytes and
332/// `a-z` ranges. A backslash is an ordinary member, as POSIX has it and as both BSD and GNU
333/// `sed` read the assignment rule's `[\"']`.
334#[derive(Debug, Clone, Copy)]
335struct Class(&'static str);
336
337impl Class {
338    fn has(self, byte: u8) -> bool {
339        let spelling = self.0.as_bytes();
340        let mut i = 0;
341        while i < spelling.len() {
342            if i + 2 < spelling.len() && spelling[i + 1] == b'-' {
343                if (spelling[i]..=spelling[i + 2]).contains(&byte) {
344                    return true;
345                }
346                i += 3;
347            } else {
348                if spelling[i] == byte {
349                    return true;
350                }
351                i += 1;
352            }
353        }
354        false
355    }
356}
357
358/// One construct of the table's extended regular expressions.
359#[derive(Debug, Clone, Copy)]
360enum Atom {
361    /// These bytes, exactly.
362    Lit(&'static str),
363    /// These letters in either case, which the table spells `[Aa][Pp][Ii]`.
364    Caseless(&'static str),
365    /// One byte of the class.
366    One(Class),
367    /// At least `min` and at most `max` bytes of the class: `?`, `*`, `+`, `{n}`, `{n,}`.
368    Run {
369        class: Class,
370        min: usize,
371        max: Option<usize>,
372    },
373    /// One of the alternatives, or nothing as well when `optional`: `(a|b)` and `(a)?`.
374    Group {
375        alternatives: &'static [&'static [Atom]],
376        optional: bool,
377    },
378}
379
380/// A credential shape: `kept` is what the replacement keeps (the `\1\2` of the assignment
381/// rule, and nothing for a table shape), `secret` what the marker replaces.
382struct Shape {
383    name: &'static str,
384    kept: &'static [Atom],
385    secret: &'static [Atom],
386}
387
388const fn run(class: &'static str, min: usize, max: Option<usize>) -> Atom {
389    Atom::Run {
390        class: Class(class),
391        min,
392        max,
393    }
394}
395
396/// `MJ_CAPTURE_SECRETS`, shape for shape and in its order.
397const SHAPES: &[Shape] = &[
398    Shape {
399        name: "anthropic-key",
400        kept: &[],
401        secret: &[Atom::Lit("sk-ant-"), run("A-Za-z0-9_-", 16, None)],
402    },
403    Shape {
404        name: "openai-key",
405        kept: &[],
406        secret: &[
407            Atom::Lit("sk-"),
408            Atom::Group {
409                alternatives: &[&[Atom::Lit("proj-")]],
410                optional: true,
411            },
412            run("A-Za-z0-9_-", 20, None),
413        ],
414    },
415    Shape {
416        name: "github-token",
417        kept: &[],
418        secret: &[
419            Atom::Lit("gh"),
420            Atom::One(Class("pousr")),
421            Atom::Lit("_"),
422            run("A-Za-z0-9_", 20, None),
423        ],
424    },
425    Shape {
426        name: "github-pat",
427        kept: &[],
428        secret: &[Atom::Lit("github_pat_"), run("A-Za-z0-9_", 20, None)],
429    },
430    Shape {
431        name: "aws-access-key-id",
432        kept: &[],
433        secret: &[Atom::Lit("AKIA"), run("0-9A-Z", 16, Some(16))],
434    },
435    Shape {
436        name: "google-api-key",
437        kept: &[],
438        secret: &[Atom::Lit("AIza"), run("0-9A-Za-z_-", 35, Some(35))],
439    },
440    Shape {
441        name: "slack-token",
442        kept: &[],
443        secret: &[
444            Atom::Lit("xox"),
445            Atom::One(Class("abprs")),
446            Atom::Lit("-"),
447            run("A-Za-z0-9-", 10, None),
448        ],
449    },
450    Shape {
451        name: "stripe-key",
452        kept: &[],
453        secret: &[
454            Atom::One(Class("rs")),
455            Atom::Lit("k_"),
456            Atom::Group {
457                alternatives: &[&[Atom::Lit("live")], &[Atom::Lit("test")]],
458                optional: false,
459            },
460            Atom::Lit("_"),
461            run("A-Za-z0-9", 16, None),
462        ],
463    },
464    Shape {
465        name: "private-key-header",
466        kept: &[],
467        secret: &[
468            Atom::Lit("-----BEGIN "),
469            Atom::Group {
470                alternatives: &[&[run("A-Z", 1, None), Atom::Lit(" ")]],
471                optional: true,
472            },
473            Atom::Lit("PRIVATE KEY-----"),
474        ],
475    },
476    Shape {
477        name: "bearer-token",
478        kept: &[],
479        secret: &[
480            Atom::One(Class("Bb")),
481            Atom::Lit("earer "),
482            run("A-Za-z0-9._~+/=-", 20, None),
483        ],
484    },
485];
486
487/// `MJ_CAPTURE_SECRET_ASSIGN`: a long opaque value assigned to something called an API key,
488/// an access token, a secret key or a password. The name and the separator are kept.
489const ASSIGNMENT: Shape = Shape {
490    name: "assignment",
491    kept: &[
492        Atom::Group {
493            alternatives: &[
494                &[
495                    Atom::Caseless("api"),
496                    run("-_", 0, Some(1)),
497                    Atom::Caseless("key"),
498                ],
499                &[
500                    Atom::Caseless("access"),
501                    run("-_", 0, Some(1)),
502                    Atom::Caseless("token"),
503                ],
504                &[
505                    Atom::Caseless("secret"),
506                    run("-_", 0, Some(1)),
507                    Atom::Caseless("key"),
508                ],
509                &[Atom::Caseless("password")],
510            ],
511            optional: false,
512        },
513        ASSIGNED,
514    ],
515    secret: &[OPAQUE_VALUE],
516};
517
518/// The assignment rule's separator: a quote, spaces, `:` or `=`, spaces and a quote.
519const ASSIGNED: Atom = Atom::Group {
520    alternatives: &[&[
521        run("\\\"'", 0, Some(1)),
522        run(" ", 0, None),
523        Atom::One(Class(":=")),
524        run(" ", 0, None),
525        run("\\\"'", 0, Some(1)),
526    ]],
527    optional: false,
528};
529
530/// The assignment rule's value: a long opaque run.
531const OPAQUE_VALUE: Atom = run("A-Za-z0-9._~+/-", 16, None);
532
533// ---------------------------------------------------------------- the matcher
534
535/// Every end at which `atoms` can match `text` from `at`, handed to `found`. A POSIX match is
536/// the longest one, so the callers keep the largest end rather than the first.
537fn ends(atoms: &[Atom], text: &[u8], at: usize, found: &mut dyn FnMut(usize)) {
538    let Some((first, rest)) = atoms.split_first() else {
539        found(at);
540        return;
541    };
542    match *first {
543        Atom::Lit(lit) => {
544            if text[at..].starts_with(lit.as_bytes()) {
545                ends(rest, text, at + lit.len(), found);
546            }
547        }
548        Atom::Caseless(word) => {
549            let end = at + word.len();
550            if end <= text.len() && text[at..end].eq_ignore_ascii_case(word.as_bytes()) {
551                ends(rest, text, end, found);
552            }
553        }
554        Atom::One(class) => {
555            if text.get(at).is_some_and(|b| class.has(*b)) {
556                ends(rest, text, at + 1, found);
557            }
558        }
559        Atom::Run { class, min, max } => {
560            let cap = max.unwrap_or(usize::MAX);
561            let mut n = 0;
562            while n < cap && text.get(at + n).is_some_and(|b| class.has(*b)) {
563                n += 1;
564            }
565            for len in (min..=n).rev() {
566                ends(rest, text, at + len, found);
567            }
568        }
569        Atom::Group {
570            alternatives,
571            optional,
572        } => {
573            if optional {
574                ends(rest, text, at, found);
575            }
576            for alternative in alternatives {
577                ends(alternative, text, at, &mut |end| {
578                    ends(rest, text, end, found)
579                });
580            }
581        }
582    }
583}
584
585impl Shape {
586    /// The longest match starting at `at`, as the end of the kept part and the end of the
587    /// whole; among equally long matches, the one whose kept part is longest, as POSIX
588    /// assigns subexpressions.
589    fn match_at(&self, text: &[u8], at: usize) -> Option<(usize, usize)> {
590        let mut best: Option<(usize, usize)> = None;
591        ends(self.kept, text, at, &mut |kept_end| {
592            ends(self.secret, text, kept_end, &mut |end| {
593                if best.is_none_or(|(k, e)| (end, kept_end) > (e, k)) {
594                    best = Some((kept_end, end));
595                }
596            });
597        });
598        best
599    }
600
601    /// Does the shape match anywhere in `text`?
602    fn found_in(&self, text: &str) -> bool {
603        let bytes = text.as_bytes();
604        (0..bytes.len()).any(|at| self.match_at(bytes, at).is_some_and(|(_, end)| end > at))
605    }
606
607    /// `s%<shape>%<kept>[redacted:<name>]%g` over the whole text, or `None` when the shape
608    /// matched nothing. Every construct of the table starts with an ASCII byte and matches
609    /// only ASCII bytes, so every cut below falls on a character boundary.
610    fn replace_all(&self, text: &str) -> Option<String> {
611        let bytes = text.as_bytes();
612        let mut out = String::with_capacity(text.len());
613        let (mut copied, mut at) = (0, 0);
614        while at < bytes.len() {
615            match self.match_at(bytes, at) {
616                Some((kept_end, end)) if end > at => {
617                    out.push_str(&text[copied..kept_end]);
618                    out.push_str(MARKER);
619                    out.push_str(self.name);
620                    out.push(']');
621                    copied = end;
622                    at = end;
623                }
624                _ => at += 1,
625            }
626        }
627        if copied == 0 {
628            return None;
629        }
630        out.push_str(&text[copied..]);
631        Some(out)
632    }
633}
634
635#[cfg(test)]
636mod tests {
637    use super::*;
638
639    /// The repository this crate is part of.
640    fn repository() -> std::path::PathBuf {
641        Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")
642    }
643
644    /// An atom as the extended regular expression `lib/capture.sh` spells it.
645    fn ere(atoms: &[Atom]) -> String {
646        atoms.iter().map(atom_ere).collect()
647    }
648
649    fn atom_ere(atom: &Atom) -> String {
650        match *atom {
651            Atom::Lit(lit) => lit.to_string(),
652            Atom::Caseless(word) => word
653                .chars()
654                .map(|c| format!("[{}{}]", c.to_ascii_uppercase(), c.to_ascii_lowercase()))
655                .collect(),
656            Atom::One(class) => format!("[{}]", class.0),
657            Atom::Run { class, min, max } => {
658                let times = match (min, max) {
659                    (0, Some(1)) => "?".to_string(),
660                    (0, None) => "*".to_string(),
661                    (1, None) => "+".to_string(),
662                    (n, None) => format!("{{{n},}}"),
663                    (n, Some(m)) if n == m => format!("{{{n}}}"),
664                    (n, Some(m)) => format!("{{{n},{m}}}"),
665                };
666                format!("[{}]{times}", class.0)
667            }
668            Atom::Group {
669                alternatives,
670                optional,
671            } => {
672                let inner: Vec<String> = alternatives.iter().map(|a| ere(a)).collect();
673                format!("({}){}", inner.join("|"), if optional { "?" } else { "" })
674            }
675        }
676    }
677
678    /// The value of a single-quoted shell assignment `NAME='...'` in `lib/capture.sh`, with
679    /// the `'\''` idiom decoded, as the shell itself reads it.
680    fn shell_value(source: &str, name: &str) -> String {
681        let opening = format!("\n{name}='");
682        let start = source
683            .find(&opening)
684            .expect("the assignment is in lib/capture.sh");
685        let mut rest = &source[start + opening.len()..];
686        let mut value = String::new();
687        loop {
688            let close = rest.find('\'').expect("the quoted value is closed");
689            value.push_str(&rest[..close]);
690            rest = &rest[close + 1..];
691            match rest.strip_prefix("\\''") {
692                Some(more) => {
693                    value.push('\'');
694                    rest = more;
695                }
696                None => return value,
697            }
698        }
699    }
700
701    #[test]
702    fn every_shape_is_the_pattern_the_shell_table_spells() {
703        let source = std::fs::read_to_string(repository().join("lib/capture.sh")).unwrap();
704        let table: Vec<(String, String)> = shell_value(&source, "MJ_CAPTURE_SECRETS")
705            .lines()
706            .map(|line| {
707                let (name, pattern) = line.split_once('\t').expect("name<TAB>ERE");
708                (name.to_string(), pattern.to_string())
709            })
710            .collect();
711        let ours: Vec<(String, String)> = SHAPES
712            .iter()
713            .map(|s| (s.name.to_string(), ere(s.kept) + ere(s.secret).as_str()))
714            .collect();
715        assert_eq!(ours, table, "the port and lib/capture.sh's table disagree");
716        assert_eq!(
717            ere(ASSIGNMENT.kept) + ere(ASSIGNMENT.secret).as_str(),
718            shell_value(&source, "MJ_CAPTURE_SECRET_ASSIGN")
719        );
720    }
721
722    #[test]
723    fn a_rendering_names_every_repetition_the_table_could_use() {
724        let bounded = [run("a", 2, Some(4)), run("b", 1, None)];
725        assert_eq!(ere(&bounded), "[a]{2,4}[b]+");
726    }
727
728    #[test]
729    fn a_class_reads_ranges_single_bytes_and_a_trailing_dash() {
730        let class = Class("A-Za-z0-9_-");
731        for byte in *b"AZaz09_-" {
732            assert!(class.has(byte), "{}", byte as char);
733        }
734        for byte in *b". /@\n" {
735            assert!(!class.has(byte), "{}", byte as char);
736        }
737        assert!(Class("\\\"'").has(b'\\'));
738        assert!(Class("-_").has(b'-'));
739    }
740
741    #[test]
742    fn an_exact_count_leaves_the_rest_of_a_longer_run() {
743        let text = format!("{}{}", "AKIA", "Q".repeat(18));
744        assert_eq!(redact_secrets(&text).text, "[redacted:aws-access-key-id]QQ");
745    }
746
747    #[test]
748    fn every_match_on_a_line_is_replaced_and_its_kind_named_once() {
749        let pat = format!("{}{}", "github_pat_", "a".repeat(20));
750        let text = format!("{pat} and {pat}\n{pat}");
751        let out = redact_secrets(&text);
752        assert_eq!(
753            out.text,
754            "[redacted:github-pat] and [redacted:github-pat]\n[redacted:github-pat]"
755        );
756        assert_eq!(out.kinds, ["github-pat"]);
757    }
758
759    #[test]
760    fn the_shapes_apply_in_table_order_so_the_narrower_name_wins() {
761        // the OpenAI shape matches an Anthropic key too; applied first, it would name it
762        let key = format!("{}{}", "sk-ant-", "a".repeat(30));
763        assert_eq!(redact_secrets(&key).text, "[redacted:anthropic-key]");
764        let project = format!("{}{}", "sk-proj-", "a".repeat(20));
765        assert_eq!(redact_secrets(&project).text, "[redacted:openai-key]");
766    }
767
768    #[test]
769    fn a_match_can_start_inside_a_word_as_the_table_has_no_boundaries() {
770        let text = format!("{}{}", "task-", "a".repeat(20));
771        assert_eq!(redact_secrets(&text).text, "ta[redacted:openai-key]");
772    }
773
774    #[test]
775    fn the_assignment_keeps_its_quotes_and_spacing() {
776        let value = "a".repeat(16);
777        let text = format!("{{\"Access_Token\" : '{value}'}}");
778        assert_eq!(
779            redact_secrets(&text).text,
780            "{\"Access_Token\" : '[redacted:assignment]'}"
781        );
782        let unquoted = format!("SecretKey={value}");
783        assert_eq!(
784            redact_secrets(&unquoted).text,
785            "SecretKey=[redacted:assignment]"
786        );
787    }
788
789    #[test]
790    fn a_header_with_one_word_or_none_is_a_private_key_and_two_words_are_not() {
791        let head = "-----BEGIN ";
792        let tail = "PRIVATE KEY-----";
793        for word in ["", "RSA ", "OPENSSH "] {
794            let text = format!("{head}{word}{tail}\nbody");
795            assert_eq!(
796                redact_secrets(&text).text,
797                "[redacted:private-key-header]\nbody"
798            );
799        }
800        let two = format!("{head}RSA ENCRYPTED {tail}");
801        assert_eq!(redact_secrets(&two).text, two);
802    }
803
804    #[test]
805    fn multibyte_text_around_a_match_survives() {
806        let key = format!("{}{}", "xoxp-", "a".repeat(10));
807        let text = format!("it’s…{key}… — done");
808        assert_eq!(
809            redact_secrets(&text).text,
810            "it’s…[redacted:slack-token]… — done"
811        );
812    }
813
814    #[test]
815    fn the_canonical_order_of_the_names_is_the_shell_s_byte_order() {
816        let mut canonical = shape_names();
817        crate::order::canonical_strings(&mut canonical);
818        let mut bytes = shape_names();
819        bytes.sort_unstable();
820        assert_eq!(
821            canonical, bytes,
822            "kinds would be listed in another order than the shell's"
823        );
824    }
825
826    #[test]
827    fn a_spelling_has_no_trailing_separator_and_the_filesystem_root_has_none() {
828        assert_eq!(spelling_of(Path::new("/srv/app/")), Some("/srv/app".into()));
829        assert_eq!(spelling_of(Path::new("/")), None);
830        assert_eq!(spelling_of(Path::new("")), None);
831    }
832
833    #[test]
834    fn a_root_that_does_not_exist_is_still_replaced_as_given() {
835        let dir = tempfile::tempdir().unwrap();
836        let gone = dir.path().join("gone");
837        let text = format!("{}/x", gone.display());
838        assert_eq!(normalise_machine_paths(&text, &gone, None), "<repo>/x");
839        assert_eq!(normalise_machine_paths(&text, Path::new("/"), None), text);
840    }
841
842    #[cfg(unix)]
843    #[test]
844    fn the_canonical_root_is_replaced_before_a_symlinked_spelling() {
845        let dir = tempfile::tempdir().unwrap();
846        let real = dir.path().join("real");
847        let link = dir.path().join("link");
848        std::fs::create_dir_all(&real).unwrap();
849        std::os::unix::fs::symlink(&real, &link).unwrap();
850        let canonical = std::fs::canonicalize(&real).unwrap();
851        let text = format!("{}/a {}/b", canonical.display(), link.display());
852        assert_eq!(
853            normalise_machine_paths(&text, &link, None),
854            "<repo>/a <repo>/b"
855        );
856    }
857
858    #[test]
859    fn every_shape_announcing_more_is_a_shape_of_the_table() {
860        let names = shape_names();
861        for (kind, _) in ANNOUNCES_MORE {
862            assert!(
863                names.contains(kind),
864                "{kind} is not a shape, so it never fires"
865            );
866        }
867    }
868
869    #[test]
870    fn a_kind_is_read_off_a_whole_marker_that_names_a_shape() {
871        let text = "[redacted:github-pat] [redacted:foo-bar] [redacted:] [redacted:Slack] \
872                    [redacted:[redacted:slack-token]] [redacted:assignment";
873        assert_eq!(kinds_in(text), ["github-pat", "slack-token"]);
874        assert!(kinds_in("nothing was taken out").is_empty());
875    }
876
877    #[test]
878    fn an_armour_line_is_found_in_any_case() {
879        assert!(contains_ignoring_case(
880            "x private key----- y",
881            "PRIVATE KEY-----"
882        ));
883        assert!(!contains_ignoring_case(
884            "PRIVATE KEY----",
885            "PRIVATE KEY-----"
886        ));
887        assert!(!contains_ignoring_case("", "PRIVATE KEY-----"));
888    }
889
890    #[test]
891    fn a_secret_access_key_is_found_only_with_a_long_value_assigned() {
892        let value = "h".repeat(16);
893        for text in [
894            format!("aws_secret_access_key = {value}"),
895            format!("export AWS_SECRET_ACCESS_KEY={value}"),
896            format!("{{\"SecretAccessKey\": \"{value}\"}}"),
897            format!("secret-access-key: '{value}'"),
898        ] {
899            assert!(SECRET_ACCESS_KEY.found_in(&text), "{text}");
900        }
901        for text in [
902            "skipped: AWS_SECRET_ACCESS_KEY is not set".to_string(),
903            format!("aws_secret_access_key = {}", "h".repeat(15)),
904            format!("aws_secret_key = {value}"),
905        ] {
906            assert!(!SECRET_ACCESS_KEY.found_in(&text), "{text}");
907        }
908    }
909
910    #[test]
911    fn public_text_refuses_what_neither_rule_can_make_safe() {
912        let dir = tempfile::tempdir().unwrap();
913        let short = format!("{}{}", "Bearer ", "a".repeat(5));
914        let refused = public_text(&short, dir.path(), None).unwrap_err();
915        assert!(refused.contains("a bearer token"), "{refused}");
916        assert!(refused.contains("`Bearer `"), "{refused}");
917        let fine = public_text("all green", dir.path(), None).unwrap();
918        assert_eq!(fine.text, "all green");
919        assert!(fine.kinds.is_empty());
920    }
921}
majordomus_cli/redaction.rs

majordomus_cli/
redaction.rs
