majordomus_cli/
scope.rs

1//! The repository scope: what a worker reads of the repository, and what it never reads.
2//!
3//! Declared once, as data: `.ai/repo/scope.yaml` when the manifest names a `scope`
4//! section, otherwise the distribution's default, the same file `majordomus init` seeds
5//! (`share/skeleton/ai/repo/scope.yaml`). `in` is the allow-list of pathspecs; `out` wins
6//! over `in` and carries a reason: a path never read, a secret, a generated asset, an
7//! archive, an image, a video, a PDF, a database dump, a fixture over its limit, a file
8//! over the limit, binary content. A path matching nothing is out as `undeclared`.
9//!
10//! The declaration is read and compiled once at start-up. Discovery drops a source outside
11//! it with a diagnostic, so the index, and every projection of the index, holds nothing a
12//! worker must not read; `repository.scope_classify` answers for any path, with the rule that decided;
13//! `repository.scope` reports the declaration and a tally of every tracked file against it.
14//! Nothing here reads a file's content beyond the first [`SNIFF_BYTES`], and the tally
15//! judges names and sizes only: content is judged when a file is read.
16
17use std::collections::BTreeMap;
18use std::path::Path;
19
20use schemars::JsonSchema;
21use serde::{Deserialize, Serialize};
22
23use crate::discovery::glob::Glob;
24use crate::error::{Error, Result};
25use crate::model::Diagnostic;
26use crate::repository::Repository;
27use crate::share::Share;
28
29/// The `version:` of `scope.yaml` this executable reads.
30pub const SCOPE_VERSION: u64 = 1;
31
32/// The file, under the tracked half, the manifest's `scope` section names by convention.
33pub const SCOPE_FILE: &str = "scope.yaml";
34
35/// The distribution's default declaration, relative to the share directory: the file
36/// `majordomus init` seeds, so that a repository declaring no scope is read under the
37/// scope it would have been given.
38pub const SKELETON_SCOPE: &str = "skeleton/ai/repo/scope.yaml";
39
40/// How many leading bytes decide whether content is text.
41pub const SNIFF_BYTES: usize = 8192;
42
43/// The pathspec every tracked file matches, for the tally.
44const EVERYTHING: &str = ":(glob)**";
45
46// ---------------------------------------------------------------- the declaration
47
48/// `scope.yaml`, typed. Unknown keys are refused by the type and by the `scope` schema.
49#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
50#[serde(deny_unknown_fields)]
51pub struct Declaration {
52    /// The format version; only [`SCOPE_VERSION`] is read.
53    pub version: u64,
54    /// The allow-list: pathspecs anchored at the repository root.
55    #[serde(rename = "in")]
56    pub r#in: Vec<String>,
57    /// What is never read; wins over `in`.
58    pub out: Out,
59}
60
61/// The `out:` mapping: every category optional, each the reason a path is out.
62#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)]
63#[serde(deny_unknown_fields)]
64pub struct Out {
65    /// Never read, by path: version control, the local half, dependencies, build outputs.
66    #[serde(default)]
67    pub paths: Vec<String>,
68    /// Content that is not text (a NUL byte in the first window) is out, whatever the name says.
69    #[serde(default)]
70    pub binary: bool,
71    /// A file over this many bytes is out, whatever it is.
72    #[serde(default, skip_serializing_if = "Option::is_none")]
73    pub max_bytes: Option<u64>,
74    /// Archives.
75    #[serde(default, skip_serializing_if = "Option::is_none")]
76    pub archive: Option<Category>,
77    /// Images.
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub image: Option<Category>,
80    /// Video.
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub video: Option<Category>,
83    /// PDF documents.
84    #[serde(default, skip_serializing_if = "Option::is_none")]
85    pub pdf: Option<Category>,
86    /// Database dumps.
87    #[serde(default, skip_serializing_if = "Option::is_none")]
88    pub database_dump: Option<Category>,
89    /// Generated assets: changed by changing the source and regenerating.
90    #[serde(default, skip_serializing_if = "Option::is_none")]
91    pub generated: Option<Category>,
92    /// Secrets: never read, never served, never quoted; a tracked one is reported.
93    #[serde(default, skip_serializing_if = "Option::is_none")]
94    pub secret: Option<Category>,
95    /// Fixtures are read up to a size; over it they are data, not context.
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub fixtures: Option<Fixtures>,
98}
99
100/// One category of `out`: pathspecs and file-name patterns.
101#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)]
102#[serde(deny_unknown_fields)]
103pub struct Category {
104    /// Pathspecs anchored at the repository root.
105    #[serde(default)]
106    pub paths: Vec<String>,
107    /// Patterns matched against the file name alone.
108    #[serde(default)]
109    pub names: Vec<String>,
110}
111
112/// The `fixtures` category: where fixtures live and how large one may be.
113#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
114#[serde(deny_unknown_fields)]
115pub struct Fixtures {
116    /// Pathspecs anchored at the repository root.
117    #[serde(default)]
118    pub paths: Vec<String>,
119    /// Patterns matched against the file name alone.
120    #[serde(default)]
121    pub names: Vec<String>,
122    /// A fixture over this many bytes is out.
123    pub max_bytes: u64,
124}
125
126impl Declaration {
127    /// Parse and validate the text of a `scope.yaml`: the version, and every pattern
128    /// well-formed (no leading slash, no `:(` prefix, no `..` segment, a name pattern
129    /// with no slash).
130    ///
131    /// ```
132    /// use majordomus_cli::scope::Declaration;
133    /// use std::path::Path;
134    /// let text = "version: 1\nin:\n  - src/**\nout:\n  paths: ['.git/']\n";
135    /// let d = Declaration::parse(Path::new("scope.yaml"), text).unwrap();
136    /// assert_eq!(d.r#in, vec!["src/**"]);
137    /// assert!(Declaration::parse(Path::new("s"), &text.replace("version: 1", "version: 2")).is_err());
138    /// assert!(Declaration::parse(Path::new("s"), &text.replace("src/**", "/src/**")).is_err());
139    /// ```
140    pub fn parse(path: &Path, text: &str) -> Result<Self> {
141        let invalid = |reason: String| Error::InvalidScope {
142            path: path.to_path_buf(),
143            reason,
144        };
145        let d: Declaration = crate::metadata::yaml::parse_into(text).map_err(invalid)?;
146        if d.version != SCOPE_VERSION {
147            return Err(invalid(format!(
148                "version {} is not {SCOPE_VERSION}",
149                d.version
150            )));
151        }
152        for (what, pattern) in d.patterns() {
153            if let Err(reason) = check_pattern(what, pattern) {
154                return Err(invalid(reason));
155            }
156        }
157        Ok(d)
158    }
159
160    /// Every pattern with where it stands, in document order: `in`, then each `out` list.
161    fn patterns(&self) -> Vec<(&'static str, &str)> {
162        let mut v: Vec<(&'static str, &str)> = Vec::new();
163        v.extend(self.r#in.iter().map(|p| ("in", p.as_str())));
164        v.extend(self.out.paths.iter().map(|p| ("out.paths", p.as_str())));
165        for (reason, c) in self.out.categories() {
166            v.extend(
167                c.paths
168                    .iter()
169                    .map(move |p| (reason.paths_key(), p.as_str())),
170            );
171            v.extend(
172                c.names
173                    .iter()
174                    .map(move |p| (reason.names_key(), p.as_str())),
175            );
176        }
177        if let Some(f) = &self.out.fixtures {
178            v.extend(f.paths.iter().map(|p| ("out.fixtures.paths", p.as_str())));
179            v.extend(f.names.iter().map(|p| ("out.fixtures.names", p.as_str())));
180        }
181        v
182    }
183}
184
185impl Out {
186    /// The name-and-path categories, in the order they decide.
187    fn categories(&self) -> Vec<(Reason, &Category)> {
188        [
189            (Reason::Secret, &self.secret),
190            (Reason::Generated, &self.generated),
191            (Reason::Archive, &self.archive),
192            (Reason::Image, &self.image),
193            (Reason::Video, &self.video),
194            (Reason::Pdf, &self.pdf),
195            (Reason::DatabaseDump, &self.database_dump),
196        ]
197        .into_iter()
198        .filter_map(|(r, c)| c.as_ref().map(|c| (r, c)))
199        .collect()
200    }
201}
202
203/// A pattern is well-formed: relative, no pathspec prefix, no parent segment; a name
204/// pattern carries no slash.
205fn check_pattern(what: &str, pattern: &str) -> std::result::Result<(), String> {
206    if pattern.is_empty() {
207        return Err(format!("{what}: an empty pattern"));
208    }
209    if pattern.starts_with('/') || pattern.starts_with(":(") {
210        return Err(format!(
211            "{what}: '{pattern}' must be relative to the repository root, with no pathspec prefix"
212        ));
213    }
214    if pattern.split('/').any(|s| s == "..") {
215        return Err(format!("{what}: '{pattern}' carries a '..' segment"));
216    }
217    if what.ends_with("names") && pattern.contains('/') {
218        return Err(format!(
219            "{what}: '{pattern}' is matched against the file name alone and may not carry a slash"
220        ));
221    }
222    Ok(())
223}
224
225// ---------------------------------------------------------------- reasons and verdicts
226
227/// Why a path is out. The order is the order the rules decide in.
228#[derive(
229    Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema,
230)]
231#[serde(rename_all = "snake_case")]
232pub enum Reason {
233    /// Named under `out.paths`.
234    Path,
235    /// A secret.
236    Secret,
237    /// A generated asset.
238    Generated,
239    /// An archive.
240    Archive,
241    /// An image.
242    Image,
243    /// Video.
244    Video,
245    /// A PDF document.
246    Pdf,
247    /// A database dump.
248    DatabaseDump,
249    /// Matches no `in` pathspec.
250    Undeclared,
251    /// A fixture over `out.fixtures.max_bytes`.
252    FixtureOverLimit,
253    /// Over `out.max_bytes`.
254    OverLimit,
255    /// Content with a NUL byte in its first [`SNIFF_BYTES`]: not text.
256    Binary,
257}
258
259impl Reason {
260    /// Every reason, in deciding order.
261    pub const ALL: [Reason; 12] = [
262        Reason::Path,
263        Reason::Secret,
264        Reason::Generated,
265        Reason::Archive,
266        Reason::Image,
267        Reason::Video,
268        Reason::Pdf,
269        Reason::DatabaseDump,
270        Reason::Undeclared,
271        Reason::FixtureOverLimit,
272        Reason::OverLimit,
273        Reason::Binary,
274    ];
275
276    /// The reason as it is written: `database_dump`, `fixture_over_limit`.
277    ///
278    /// ```
279    /// use majordomus_cli::scope::Reason;
280    /// assert_eq!(Reason::DatabaseDump.as_str(), "database_dump");
281    /// assert_eq!(Reason::ALL.len(), 12);
282    /// ```
283    pub fn as_str(self) -> &'static str {
284        match self {
285            Reason::Path => "path",
286            Reason::Secret => "secret",
287            Reason::Generated => "generated",
288            Reason::Archive => "archive",
289            Reason::Image => "image",
290            Reason::Video => "video",
291            Reason::Pdf => "pdf",
292            Reason::DatabaseDump => "database_dump",
293            Reason::Undeclared => "undeclared",
294            Reason::FixtureOverLimit => "fixture_over_limit",
295            Reason::OverLimit => "over_limit",
296            Reason::Binary => "binary",
297        }
298    }
299
300    fn paths_key(self) -> &'static str {
301        match self {
302            Reason::Secret => "out.secret.paths",
303            Reason::Generated => "out.generated.paths",
304            Reason::Archive => "out.archive.paths",
305            Reason::Image => "out.image.paths",
306            Reason::Video => "out.video.paths",
307            Reason::Pdf => "out.pdf.paths",
308            Reason::DatabaseDump => "out.database_dump.paths",
309            _ => "out.paths",
310        }
311    }
312
313    fn names_key(self) -> &'static str {
314        match self {
315            Reason::Secret => "out.secret.names",
316            Reason::Generated => "out.generated.names",
317            Reason::Archive => "out.archive.names",
318            Reason::Image => "out.image.names",
319            Reason::Video => "out.video.names",
320            Reason::Pdf => "out.pdf.names",
321            Reason::DatabaseDump => "out.database_dump.names",
322            _ => "out.names",
323        }
324    }
325}
326
327impl std::fmt::Display for Reason {
328    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
329        f.write_str(self.as_str())
330    }
331}
332
333/// In or out.
334#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
335#[serde(rename_all = "lowercase")]
336pub enum Verdict {
337    /// Read.
338    In,
339    /// Never read; the reason says why.
340    Out,
341}
342
343/// Where the declaration was read from.
344#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
345#[serde(rename_all = "lowercase")]
346pub enum Origin {
347    /// The repository's own, named by the manifest's `scope` section.
348    Repository,
349    /// The distribution's default, because the repository declares none.
350    Distribution,
351}
352
353/// One path, judged.
354#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
355pub struct Classification {
356    /// The path, repository-relative, as judged.
357    pub path: String,
358    /// In or out.
359    pub verdict: Verdict,
360    /// Why it is out; absent when it is in.
361    #[serde(skip_serializing_if = "Option::is_none")]
362    pub reason: Option<Reason>,
363    /// The pattern or limit that decided: the `in` pathspec, the `out` pattern, `binary`,
364    /// `max_bytes` or `fixtures.max_bytes`; absent for `undeclared`.
365    #[serde(skip_serializing_if = "Option::is_none")]
366    pub rule: Option<String>,
367    /// Whether the path exists in the work tree; a path that does not is judged by name.
368    pub exists: bool,
369    /// Whether the path is a directory; a directory is in when something beneath it can be.
370    pub directory: bool,
371    /// The size, when the path is an existing file.
372    #[serde(skip_serializing_if = "Option::is_none")]
373    pub bytes: Option<u64>,
374}
375
376impl Classification {
377    fn out(path: &str, reason: Reason, rule: Option<String>) -> Self {
378        Classification {
379            path: path.to_string(),
380            verdict: Verdict::Out,
381            reason: Some(reason),
382            rule,
383            exists: false,
384            directory: false,
385            bytes: None,
386        }
387    }
388
389    fn r#in(path: &str, rule: String) -> Self {
390        Classification {
391            path: path.to_string(),
392            verdict: Verdict::In,
393            reason: None,
394            rule: Some(rule),
395            exists: false,
396            directory: false,
397            bytes: None,
398        }
399    }
400
401    /// Is the path in?
402    pub fn is_in(&self) -> bool {
403        self.verdict == Verdict::In
404    }
405}
406
407// ---------------------------------------------------------------- the compiled scope
408
409struct Rule {
410    reason: Reason,
411    text: String,
412    glob: Glob,
413}
414
415fn compile_pathspec(text: &str) -> Glob {
416    if let Some(dir) = text.strip_suffix('/') {
417        Glob::new(&format!("{dir}/**"))
418    } else {
419        Glob::new(text)
420    }
421}
422
423fn rules<'a>(reason: Reason, patterns: impl Iterator<Item = &'a String>) -> Vec<Rule> {
424    patterns
425        .map(|p| Rule {
426            reason,
427            text: p.clone(),
428            glob: compile_pathspec(p),
429        })
430        .collect()
431}
432
433fn name_rules<'a>(reason: Reason, patterns: impl Iterator<Item = &'a String>) -> Vec<Rule> {
434    patterns
435        .map(|p| Rule {
436            reason,
437            text: p.clone(),
438            glob: Glob::new(p),
439        })
440        .collect()
441}
442
443/// The declaration, compiled, with where it came from.
444pub struct Scope {
445    declaration: Declaration,
446    origin: Origin,
447    path: String,
448    in_rules: Vec<Rule>,
449    path_rules: Vec<Rule>,
450    name_rules: Vec<Rule>,
451    fixture_paths: Vec<Rule>,
452    fixture_names: Vec<Rule>,
453}
454
455impl std::fmt::Debug for Scope {
456    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
457        f.debug_struct("Scope")
458            .field("origin", &self.origin)
459            .field("path", &self.path)
460            .field("in", &self.in_rules.len())
461            .field("out", &(self.path_rules.len() + self.name_rules.len()))
462            .finish()
463    }
464}
465
466impl Scope {
467    /// Compile a declaration read from `path` (repository-relative, or the share path).
468    pub fn compile(declaration: Declaration, origin: Origin, path: &str) -> Self {
469        let out = &declaration.out;
470        let in_rules = rules(Reason::Undeclared, declaration.r#in.iter());
471        let mut path_rules = rules(Reason::Path, out.paths.iter());
472        let mut name_rules = Vec::new();
473        for (reason, c) in out.categories() {
474            path_rules.extend(rules(reason, c.paths.iter()));
475            name_rules.extend(name_rules_of(reason, c));
476        }
477        let (fixture_paths, fixture_names) = match &out.fixtures {
478            Some(f) => (
479                rules(Reason::FixtureOverLimit, f.paths.iter()),
480                name_rules_of(
481                    Reason::FixtureOverLimit,
482                    &Category {
483                        paths: Vec::new(),
484                        names: f.names.clone(),
485                    },
486                ),
487            ),
488            None => (Vec::new(), Vec::new()),
489        };
490        Scope {
491            declaration,
492            origin,
493            path: path.to_string(),
494            in_rules,
495            path_rules,
496            name_rules,
497            fixture_paths,
498            fixture_names,
499        }
500    }
501
502    /// Read the repository's declaration when its manifest names a `scope` section,
503    /// otherwise the distribution's default under the share directory.
504    pub fn load(share: &Share, repo: &Repository) -> Result<Self> {
505        if let Some(section) = repo.section_path("scope") {
506            let abs = repo.root().join(&section);
507            let text = std::fs::read_to_string(&abs).map_err(|e| Error::io(&abs, e))?;
508            let d = Declaration::parse(&abs, &text)?;
509            return Ok(Self::compile(d, Origin::Repository, &section));
510        }
511        let abs = share.skeleton_scope();
512        let text = std::fs::read_to_string(&abs).map_err(|e| Error::InvalidScope {
513            path: abs.clone(),
514            reason: format!(
515                "the distribution's default scope cannot be read ({e}); the manifest of {} names no `scope` section, so the default is what applies",
516                repo.root().display()
517            ),
518        })?;
519        let d = Declaration::parse(&abs, &text)?;
520        Ok(Self::compile(
521            d,
522            Origin::Distribution,
523            &abs.display().to_string(),
524        ))
525    }
526
527    /// The declaration as read.
528    pub fn declaration(&self) -> &Declaration {
529        &self.declaration
530    }
531
532    /// Where the declaration came from.
533    pub fn origin(&self) -> Origin {
534        self.origin
535    }
536
537    /// The file the declaration was read from: repository-relative for the repository's
538    /// own, the share path for the distribution's.
539    pub fn path(&self) -> &str {
540        &self.path
541    }
542
543    /// Judge a repository-relative path by its name alone: the `out` paths and names, then
544    /// the `in` pathspecs. Size and content are not consulted; [`Scope::classify`] does.
545    /// A directory is in when an `in` pathspec can match beneath it.
546    ///
547    /// ```
548    /// use majordomus_cli::scope::{Declaration, Origin, Reason, Scope, Verdict};
549    /// use std::path::Path;
550    /// let text = "version: 1\nin:\n  - src/**\n  - docs/**\nout:\n  paths: ['.git/', '**/target/']\n  secret:\n    names: ['.env', '*.pem']\n";
551    /// let scope = Scope::compile(Declaration::parse(Path::new("s"), text).unwrap(), Origin::Repository, "s");
552    /// assert_eq!(scope.classify_name("src/main.rs", false).verdict, Verdict::In);
553    /// assert_eq!(scope.classify_name("src/.env", false).reason, Some(Reason::Secret));
554    /// assert_eq!(scope.classify_name("src/target/x.rs", false).reason, Some(Reason::Path));
555    /// assert_eq!(scope.classify_name("README.md", false).reason, Some(Reason::Undeclared));
556    /// assert_eq!(scope.classify_name("docs", true).verdict, Verdict::In);
557    /// ```
558    pub fn classify_name(&self, path: &str, directory: bool) -> Classification {
559        let name = path.rsplit('/').next().unwrap_or(path);
560        for r in &self.path_rules {
561            if r.glob.matches(path) {
562                let mut c = Classification::out(path, r.reason, Some(r.text.clone()));
563                c.directory = directory;
564                return c;
565            }
566        }
567        if !directory {
568            for r in &self.name_rules {
569                if r.glob.matches(name) {
570                    return Classification::out(path, r.reason, Some(r.text.clone()));
571                }
572            }
573        }
574        let admitted = self.in_rules.iter().find(|r| {
575            if directory {
576                r.glob.could_match_under(path)
577            } else {
578                r.glob.matches(path)
579            }
580        });
581        let mut c = match admitted {
582            Some(r) => Classification::r#in(path, r.text.clone()),
583            None => Classification::out(path, Reason::Undeclared, None),
584        };
585        c.directory = directory;
586        c
587    }
588
589    /// Is the path a fixture, by path or by name? The rule that says so.
590    fn fixture_rule(&self, path: &str) -> Option<&Rule> {
591        let name = path.rsplit('/').next().unwrap_or(path);
592        self.fixture_paths
593            .iter()
594            .find(|r| r.glob.matches(path))
595            .or_else(|| self.fixture_names.iter().find(|r| r.glob.matches(name)))
596    }
597
598    /// Judge a path the name admits by its size: a fixture over its limit, a file over
599    /// the limit. `None` when the size changes nothing.
600    fn judge_size(&self, path: &str, bytes: u64) -> Option<Classification> {
601        if let (Some(f), Some(rule)) = (&self.declaration.out.fixtures, self.fixture_rule(path)) {
602            if bytes > f.max_bytes {
603                return Some(Classification::out(
604                    path,
605                    Reason::FixtureOverLimit,
606                    Some(format!(
607                        "{} over fixtures.max_bytes {}",
608                        rule.text, f.max_bytes
609                    )),
610                ));
611            }
612        }
613        if let Some(max) = self.declaration.out.max_bytes {
614            if bytes > max {
615                return Some(Classification::out(
616                    path,
617                    Reason::OverLimit,
618                    Some(format!("max_bytes {max}")),
619                ));
620            }
621        }
622        None
623    }
624
625    /// Judge a repository-relative path by name, then, when it exists as a file the name
626    /// admits, by size and by the first [`SNIFF_BYTES`] of its content. A symlink is judged
627    /// by name only and reported as not existing: sources are regular files.
628    pub fn classify(&self, root: &Path, path: &str) -> Classification {
629        let abs = root.join(path);
630        let meta = std::fs::symlink_metadata(&abs).ok();
631        let is_dir = meta.as_ref().is_some_and(|m| m.is_dir());
632        let is_file = meta.as_ref().is_some_and(|m| m.is_file());
633        let mut c = self.classify_name(path, is_dir);
634        c.exists = is_dir || is_file;
635        if !is_file {
636            return c;
637        }
638        let bytes = meta.map(|m| m.len()).unwrap_or(0);
639        c.bytes = Some(bytes);
640        if !c.is_in() {
641            return c;
642        }
643        if let Some(mut out) = self.judge_size(path, bytes) {
644            out.exists = true;
645            out.bytes = Some(bytes);
646            return out;
647        }
648        if self.declaration.out.binary && !is_text(&abs) {
649            let mut out = Classification::out(path, Reason::Binary, Some("binary".into()));
650            out.exists = true;
651            out.bytes = Some(bytes);
652            return out;
653        }
654        c
655    }
656
657    /// Every tracked file judged by name and size, counted; the out ones listed with
658    /// their reason. Content is not read: a binary is caught here only by its name.
659    pub fn tally(&self, root: &Path, files: &[String]) -> Tally {
660        let mut tally = Tally {
661            files: files.len(),
662            r#in: 0,
663            out: 0,
664            by_reason: BTreeMap::new(),
665            out_files: Vec::new(),
666        };
667        let sized =
668            self.declaration.out.max_bytes.is_some() || self.declaration.out.fixtures.is_some();
669        for path in files {
670            let mut c = self.classify_name(path, false);
671            if c.is_in() && sized {
672                if let Ok(meta) = std::fs::symlink_metadata(root.join(path)) {
673                    if meta.is_file() {
674                        if let Some(out) = self.judge_size(path, meta.len()) {
675                            c = out;
676                        }
677                    }
678                }
679            }
680            match c.reason {
681                None => tally.r#in += 1,
682                Some(reason) => {
683                    tally.out += 1;
684                    *tally
685                        .by_reason
686                        .entry(reason.as_str().to_string())
687                        .or_insert(0) += 1;
688                    tally.out_files.push(OutFile {
689                        path: path.clone(),
690                        reason,
691                        rule: c.rule,
692                    });
693                }
694            }
695        }
696        tally
697    }
698
699    /// The tracked files of the repository, through `source`, judged: the tally, and a
700    /// diagnostic for every tracked secret, which is never read and should not be tracked.
701    pub fn tally_repository(
702        &self,
703        repo: &Repository,
704        source: &dyn crate::discovery::DiscoverySource,
705    ) -> Result<(Tally, Vec<Diagnostic>)> {
706        let mut files = source.enumerate(repo.root(), EVERYTHING)?;
707        files.sort();
708        files.dedup();
709        let tally = self.tally(repo.root(), &files);
710        let diagnostics = tally
711            .out_files
712            .iter()
713            .filter(|f| f.reason == Reason::Secret)
714            .map(|f| {
715                Diagnostic::warning(
716                    "tracked_secret",
717                    Some(f.path.clone()),
718                    format!(
719                        "a secret is tracked ({}); it is never read or served, and it should not be in the repository",
720                        f.rule.as_deref().unwrap_or("secret")
721                    ),
722                )
723            })
724            .collect();
725        Ok((tally, diagnostics))
726    }
727}
728
729fn name_rules_of(reason: Reason, c: &Category) -> Vec<Rule> {
730    name_rules(reason, c.names.iter())
731}
732
733/// Is the file text? The first [`SNIFF_BYTES`] carry no NUL byte, which is the judgement
734/// version control makes too; encoding is the reader's concern, so a Latin-1 file passes
735/// here and is refused by name where it is read. An unreadable file is not text.
736pub fn is_text(abs: &Path) -> bool {
737    use std::io::Read;
738    let Ok(mut f) = std::fs::File::open(abs) else {
739        return false;
740    };
741    let mut buf = vec![0u8; SNIFF_BYTES];
742    let mut read = 0;
743    while read < buf.len() {
744        match f.read(&mut buf[read..]) {
745            Ok(0) => break,
746            Ok(n) => read += n,
747            Err(_) => return false,
748        }
749    }
750    is_text_bytes(&buf[..read])
751}
752
753/// The text judgement over a window of bytes: no NUL byte.
754///
755/// ```
756/// use majordomus_cli::scope::is_text_bytes;
757/// assert!(is_text_bytes(b"version: 1\n"));
758/// assert!(is_text_bytes("příručka".as_bytes()));
759/// assert!(is_text_bytes(b"caf\xe9"), "an encoding is not a binary; the reader refuses it");
760/// assert!(!is_text_bytes(b"\x89PNG\r\n\x1a\n\x00"));
761/// ```
762pub fn is_text_bytes(bytes: &[u8]) -> bool {
763    !bytes.contains(&0)
764}
765
766// ---------------------------------------------------------------- the tally
767
768/// Every tracked file against the scope, counted.
769#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
770pub struct Tally {
771    /// How many tracked files were judged.
772    pub files: usize,
773    /// How many are in.
774    #[serde(rename = "in")]
775    pub r#in: usize,
776    /// How many are out.
777    pub out: usize,
778    /// How many are out for each reason present.
779    pub by_reason: BTreeMap<String, usize>,
780    /// Every tracked file that is out, with its reason and the rule that decided.
781    pub out_files: Vec<OutFile>,
782}
783
784/// One tracked file that is out.
785#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
786pub struct OutFile {
787    /// Repository-relative.
788    pub path: String,
789    /// Why.
790    pub reason: Reason,
791    /// The rule that decided; absent for `undeclared`.
792    #[serde(skip_serializing_if = "Option::is_none")]
793    pub rule: Option<String>,
794}
795
796/// The scope with its tally: what the index was built under.
797#[derive(Debug)]
798pub struct Scoped {
799    /// The compiled declaration.
800    pub scope: Scope,
801    /// Every tracked file judged.
802    pub tally: Tally,
803}
804
805impl Default for Scoped {
806    /// A scope admitting nothing and a tally of nothing: for an index built by hand in a
807    /// test or a doctest, never by the application.
808    fn default() -> Self {
809        Scoped {
810            scope: Scope::compile(
811                Declaration {
812                    version: SCOPE_VERSION,
813                    r#in: Vec::new(),
814                    out: Out::default(),
815                },
816                Origin::Distribution,
817                "",
818            ),
819            tally: Tally {
820                files: 0,
821                r#in: 0,
822                out: 0,
823                by_reason: BTreeMap::new(),
824                out_files: Vec::new(),
825            },
826        }
827    }
828}
829
830#[cfg(test)]
831mod tests {
832    use super::*;
833
834    const TEXT: &str = "version: 1
835in:
836  - .ai/repo/**
837  - src/**
838  - test/**
839  - docs/**
840  - README.md
841out:
842  paths:
843    - .git/
844    - .ai/local/
845    - '**/target/'
846    - docs/generated/
847  binary: true
848  max_bytes: 100
849  image:
850    names: ['*.png', '*.svg']
851  generated:
852    paths: [site/data/generated/]
853    names: ['*.min.js']
854  secret:
855    names: ['.env', '.env.*', '*.pem']
856  fixtures:
857    paths: ['**/fixtures/']
858    names: ['*.snap']
859    max_bytes: 10
860";
861
862    fn scope() -> Scope {
863        Scope::compile(
864            Declaration::parse(Path::new("s"), TEXT).unwrap(),
865            Origin::Repository,
866            ".ai/repo/scope.yaml",
867        )
868    }
869
870    #[test]
871    fn out_wins_over_in_and_undeclared_is_out() {
872        let s = scope();
873        let c = s.classify_name("src/main.rs", false);
874        assert!(c.is_in());
875        assert_eq!(c.rule.as_deref(), Some("src/**"));
876        assert_eq!(
877            s.classify_name(".ai/local/state/current.yaml", false)
878                .reason,
879            Some(Reason::Path)
880        );
881        assert_eq!(
882            s.classify_name("src/target/debug/x", false).reason,
883            Some(Reason::Path)
884        );
885        assert_eq!(
886            s.classify_name("docs/generated/openapi.json", false).reason,
887            Some(Reason::Path),
888            "a category path is a path rule with the category's reason only when listed there"
889        );
890        assert_eq!(
891            s.classify_name("site/data/generated/x.json", false).reason,
892            Some(Reason::Generated)
893        );
894        assert_eq!(
895            s.classify_name("src/.env", false).reason,
896            Some(Reason::Secret)
897        );
898        assert_eq!(
899            s.classify_name("src/.env.local", false).reason,
900            Some(Reason::Secret)
901        );
902        assert_eq!(
903            s.classify_name("docs/logo.png", false).reason,
904            Some(Reason::Image)
905        );
906        assert_eq!(
907            s.classify_name("src/app.min.js", false).reason,
908            Some(Reason::Generated)
909        );
910        let c = s.classify_name("Cargo.toml", false);
911        assert_eq!(c.reason, Some(Reason::Undeclared));
912        assert_eq!(c.rule, None);
913    }
914
915    #[test]
916    fn directories_are_in_when_something_beneath_can_be() {
917        let s = scope();
918        assert!(s.classify_name("src", true).is_in());
919        assert!(s.classify_name("docs", true).is_in());
920        assert_eq!(
921            s.classify_name("docs/generated", true).reason,
922            Some(Reason::Path)
923        );
924        assert_eq!(s.classify_name(".git", true).reason, Some(Reason::Path));
925        assert_eq!(
926            s.classify_name("node_modules", true).reason,
927            Some(Reason::Undeclared)
928        );
929    }
930
931    #[test]
932    fn size_and_content_decide_after_the_name() {
933        let dir = tempfile::tempdir().unwrap();
934        let root = dir.path();
935        let write = |rel: &str, bytes: &[u8]| {
936            let p = root.join(rel);
937            std::fs::create_dir_all(p.parent().unwrap()).unwrap();
938            std::fs::write(p, bytes).unwrap();
939        };
940        write("src/small.rs", b"fn main() {}\n");
941        write("src/big.rs", &[b'a'; 101]);
942        write("src/blob.rs", b"\x00\x01\x02");
943        write("test/fixtures/ok.json", b"{}");
944        write("test/fixtures/large.json", &[b'x'; 11]);
945        write("test/one.snap", &[b'x'; 11]);
946        let s = scope();
947        let c = s.classify(root, "src/small.rs");
948        assert!(c.is_in() && c.exists && c.bytes == Some(13));
949        assert_eq!(
950            s.classify(root, "src/big.rs").reason,
951            Some(Reason::OverLimit)
952        );
953        assert_eq!(s.classify(root, "src/blob.rs").reason, Some(Reason::Binary));
954        assert!(s.classify(root, "test/fixtures/ok.json").is_in());
955        let c = s.classify(root, "test/fixtures/large.json");
956        assert_eq!(c.reason, Some(Reason::FixtureOverLimit));
957        assert!(c.rule.unwrap().contains("fixtures.max_bytes 10"));
958        assert_eq!(
959            s.classify(root, "test/one.snap").reason,
960            Some(Reason::FixtureOverLimit)
961        );
962        let c = s.classify(root, "src/absent.rs");
963        assert!(c.is_in() && !c.exists && c.bytes.is_none());
964        let c = s.classify(root, "src");
965        assert!(c.is_in() && c.exists && c.directory);
966    }
967
968    #[test]
969    fn tally_counts_names_and_sizes() {
970        let dir = tempfile::tempdir().unwrap();
971        let root = dir.path();
972        std::fs::create_dir_all(root.join("src")).unwrap();
973        std::fs::write(root.join("src/big.rs"), vec![b'a'; 101]).unwrap();
974        let files = vec![
975            "src/a.rs".to_string(),
976            "src/big.rs".to_string(),
977            "docs/x.png".to_string(),
978            "src/.env".to_string(),
979            "Cargo.toml".to_string(),
980        ];
981        let s = scope();
982        let t = s.tally(root, &files);
983        assert_eq!((t.files, t.r#in, t.out), (5, 1, 4));
984        assert_eq!(t.by_reason["image"], 1);
985        assert_eq!(t.by_reason["secret"], 1);
986        assert_eq!(t.by_reason["undeclared"], 1);
987        assert_eq!(t.by_reason["over_limit"], 1);
988        assert_eq!(t.out_files.len(), 4);
989    }
990
991    #[test]
992    fn declaration_refuses_bad_shapes() {
993        let bad = |t: &str| {
994            assert!(
995                matches!(
996                    Declaration::parse(Path::new("s"), t),
997                    Err(Error::InvalidScope { .. })
998                ),
999                "{t}"
1000            )
1001        };
1002        bad(&TEXT.replace("version: 1", "version: 2"));
1003        bad(&TEXT.replace("  - src/**", "  - /src/**"));
1004        bad(&TEXT.replace("  - src/**", "  - ':(glob)src/**'"));
1005        bad(&TEXT.replace("  - src/**", "  - src/../**"));
1006        bad(&TEXT.replace("'*.png'", "'img/*.png'"));
1007        bad(&format!("{TEXT}  colour: red\n"));
1008        bad("version: 1\nin: []\n");
1009    }
1010}
majordomus_cli/scope.rs

majordomus_cli/
scope.rs
