majordomus_cli/discovery/mod.rs
1//! Discovery: which files of the repository are declarative sources, and of which kind.
2//!
3//! Nothing here walks `.ai/` looking for interesting files. The repository declares its
4//! source classes in `sources.yaml` under the manifest's `knowledge` section, each class a
5//! pathspec and a kind, and discovery enumerates exactly those. Two implementations of the
6//! enumeration exist: the version-control index, which the contract prescribes, and a
7//! filesystem walk with the same glob semantics for a checkout git cannot describe.
8//!
9//! # Why the declaration and not a walk
10//!
11//! A walk of `.ai/` would have to guess what a file is from where it sits, which makes the
12//! directory layout part of the contract and makes adding a file a change to the reader.
13//! Declaring the classes instead means a repository can add a kind — a pathspec and a kind
14//! name in `sources.yaml`, a schema under `.ai/repo/knowledge/` — and this executable
15//! serves it without being rebuilt.
16//!
17//! # The two enumerations
18//!
19//! [`VcsIndex`] asks git for the tracked files, which is the layer's contract and the only
20//! mode [`DiscoveryKind`] declares: what is not tracked is not part of the layer. The one
21//! layer that sentence cannot describe is a layer none of which is tracked — a personal
22//! layer (ADR 0145), or one created a minute ago and not committed yet: its tracked set is
23//! empty by construction, so [`layer_is_untracked`] says so and the work tree is read.
24//! [`FileSystem`] walks the tree with the same glob semantics, for a checkout git cannot
25//! describe — a scratch fixture, a tree with no commits. The index records which one it
26//! used, so an answer can
27//! never be silently about a different file set than the caller expected.
28//!
29//! ```
30//! use majordomus_cli::discovery::{DiscoveryKind, Sources};
31//!
32//! let declaration = "\
33//! version: 1
34//! sources:
35//! - id: rules
36//! kind: rule
37//! discovery: vcs
38//! pathspec: ':(glob).ai/repo/rules/**/*.md'
39//! required: false
40//! ";
41//! let sources = Sources::parse(std::path::Path::new("sources.yaml"), declaration).unwrap();
42//! assert_eq!(sources.sources.len(), 1);
43//! assert_eq!(sources.sources[0].kind, "rule");
44//! assert_eq!(sources.sources[0].discovery, DiscoveryKind::Vcs);
45//! ```
46
47pub mod glob;
48
49use std::path::Path;
50
51use serde::{Deserialize, Serialize};
52
53use crate::error::{Error, Result};
54use crate::git;
55use crate::model::Diagnostic;
56use crate::repository::Repository;
57
58/// The file, under the `knowledge` section, that declares the source classes.
59pub const SOURCES_FILE: &str = "sources.yaml";
60
61/// The `version:` of `sources.yaml` this executable reads.
62pub const SOURCES_VERSION: u64 = 1;
63
64/// The prefix every pathspec carries: `*` never crosses a directory separator.
65pub const GLOB_PREFIX: &str = ":(glob)";
66
67/// The kind whose class the manifest's `context.documents` names are reserved for.
68pub const CONTEXT_KIND: &str = "context";
69
70#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
71#[serde(rename_all = "lowercase")]
72/// How a class enumerates its files; `vcs` is the only discovery the contract knows.
73pub enum DiscoveryKind {
74 /// Through the version-control index: tracked files only.
75 Vcs,
76}
77
78/// One class of `sources.yaml`.
79#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
80#[serde(deny_unknown_fields)]
81pub struct SourceClass {
82 /// The class, unique; also the order sources are reported in.
83 pub id: String,
84 /// The kind every source in this class produces.
85 pub kind: String,
86 /// How the files are enumerated.
87 pub discovery: DiscoveryKind,
88 /// One `:(glob)` pathspec, relative to the repository root.
89 pub pathspec: String,
90 /// Must a repository with this layer have at least one file of this class?
91 pub required: bool,
92}
93
94#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
95#[serde(deny_unknown_fields)]
96/// `sources.yaml`, typed: the classes in declared order.
97pub struct Sources {
98 /// The format version; only [`SOURCES_VERSION`] is read.
99 pub version: u64,
100 /// The classes, in the order they are reported.
101 pub sources: Vec<SourceClass>,
102}
103
104impl Sources {
105 /// Read `sources.yaml` from the repository's `knowledge` section.
106 pub fn load(repo: &Repository) -> Result<Self> {
107 let section = repo
108 .section_path("knowledge")
109 .ok_or_else(|| Error::InvalidManifest {
110 path: repo.root().join(crate::repository::MANIFEST),
111 reason: "no 'knowledge' section, so no sources can be declared".into(),
112 })?;
113 let path = repo.root().join(§ion).join(SOURCES_FILE);
114 let text = std::fs::read_to_string(&path).map_err(|e| Error::io(&path, e))?;
115 Self::parse(&path, &text)
116 }
117
118 /// Parse and validate the text of a `sources.yaml`.
119 ///
120 /// ```
121 /// use majordomus_cli::discovery::Sources;
122 /// use std::path::Path;
123 /// let text = "version: 1\nsources:\n - id: rule\n kind: rule\n discovery: vcs\n pathspec: ':(glob).ai/repo/rules/**/*.md'\n required: true\n";
124 /// let sources = Sources::parse(Path::new("sources.yaml"), text).unwrap();
125 /// assert_eq!(sources.sources[0].kind, "rule");
126 /// assert!(Sources::parse(Path::new("s"), &text.replace("version: 1", "version: 2")).is_err());
127 /// ```
128 pub fn parse(path: &Path, text: &str) -> Result<Self> {
129 let sources: Sources =
130 crate::metadata::yaml::parse_into(text).map_err(|reason| Error::InvalidSources {
131 path: path.to_path_buf(),
132 reason,
133 })?;
134 if sources.version != SOURCES_VERSION {
135 return Err(Error::InvalidSources {
136 path: path.to_path_buf(),
137 reason: format!("version {} is not {SOURCES_VERSION}", sources.version),
138 });
139 }
140 let mut seen = std::collections::BTreeSet::new();
141 for class in &sources.sources {
142 if !seen.insert(class.id.as_str()) {
143 return Err(Error::InvalidSources {
144 path: path.to_path_buf(),
145 reason: format!("class '{}' is declared twice", class.id),
146 });
147 }
148 if !class.pathspec.starts_with(GLOB_PREFIX) {
149 return Err(Error::InvalidSources {
150 path: path.to_path_buf(),
151 reason: format!(
152 "class '{}': pathspec must start with {GLOB_PREFIX}",
153 class.id
154 ),
155 });
156 }
157 }
158 Ok(sources)
159 }
160}
161
162/// A file discovered through a class, before it is read.
163#[derive(Debug, Clone, PartialEq, Eq)]
164pub struct DiscoveredFile {
165 /// The class that discovered the file.
166 pub class: String,
167 /// The kind the class declares.
168 pub kind: String,
169 /// Repository-relative, forward slashes.
170 pub rel_path: String,
171}
172
173/// How the files matching a pathspec are enumerated. The trait exists because two
174/// enumerations are shipped and the caller chooses one explicitly.
175pub trait DiscoverySource {
176 /// A stable name for diagnostics: `vcs` or `filesystem`.
177 fn name(&self) -> &'static str;
178 /// Repository-relative paths matching `pathspec` (with its `:(glob)` prefix).
179 fn enumerate(&self, root: &Path, pathspec: &str) -> Result<Vec<String>>;
180}
181
182/// The version-control index: tracked files only, as the contract prescribes. The index
183/// is read from git once per build (`git ls-files`, every tracked path) and each class's
184/// `:(glob)` pathspec is matched in process with the same matcher the filesystem walk
185/// uses; one subprocess per repository instead of one per class.
186#[derive(Default)]
187pub struct VcsIndex {
188 tracked: std::cell::OnceCell<Vec<String>>,
189}
190
191impl DiscoverySource for VcsIndex {
192 fn name(&self) -> &'static str {
193 "vcs"
194 }
195 fn enumerate(&self, root: &Path, pathspec: &str) -> Result<Vec<String>> {
196 let tracked = match self.tracked.get() {
197 Some(t) => t,
198 None => {
199 let all = git::ls_files_all(root)?;
200 let _ = self.tracked.set(all);
201 self.tracked.get().expect("just set")
202 }
203 };
204 let pattern = pathspec.strip_prefix(GLOB_PREFIX).unwrap_or(pathspec);
205 let matcher = glob::Glob::new(pattern);
206 let mut out: Vec<String> = tracked
207 .iter()
208 .filter(|p| matcher.matches(p))
209 .cloned()
210 .collect();
211 crate::order::canonical(&mut out);
212 Ok(out)
213 }
214}
215
216/// Whether git has never been given the layer under `root`: its manifest is on disk and the
217/// index holds no entry for it.
218///
219/// The manifest stands for the layer because nothing reads a layer without it, and a
220/// repository that tracks its layer tracks that file first. It is asked of git each time
221/// rather than recorded anywhere, so a layer that is committed later is read through the
222/// index from that moment and nobody has to change a setting. A repository git cannot
223/// describe answers `false`: the caller's own handling of an unavailable git stands.
224///
225/// ```
226/// use majordomus_cli::discovery::layer_is_untracked;
227/// use std::process::Command;
228///
229/// let dir = tempfile::tempdir().unwrap();
230/// let git = |args: &[&str]| {
231/// Command::new("git").arg("-C").arg(dir.path()).args(args).output().unwrap()
232/// };
233/// git(&["init", "-q"]);
234/// std::fs::create_dir(dir.path().join(".ai")).unwrap();
235/// std::fs::write(dir.path().join(".ai/manifest.yaml"), "schema: manifest/v1\n").unwrap();
236/// assert!(layer_is_untracked(dir.path()), "on disk, and git was never given it");
237///
238/// git(&["add", ".ai/manifest.yaml"]);
239/// assert!(!layer_is_untracked(dir.path()), "staged is tracked: the index holds it");
240///
241/// let bare = tempfile::tempdir().unwrap();
242/// assert!(!layer_is_untracked(bare.path()), "no manifest, no layer to call untracked");
243/// ```
244pub fn layer_is_untracked(root: &Path) -> bool {
245 root.join(crate::repository::MANIFEST).is_file()
246 && matches!(
247 git::ls_files(root, crate::repository::MANIFEST),
248 Ok(tracked) if tracked.is_empty()
249 )
250}
251
252/// A walk of the work tree with `:(glob)` semantics: for a checkout that is not a git work
253/// tree, or when untracked files are wanted deliberately. Symlinks are never followed and
254/// the checkout-local half of the layer is never a source.
255pub struct FileSystem {
256 /// Repository-relative directory prefixes never entered (`.git`, the local half).
257 pub excluded: Vec<String>,
258}
259
260impl DiscoverySource for FileSystem {
261 fn name(&self) -> &'static str {
262 "filesystem"
263 }
264 fn enumerate(&self, root: &Path, pathspec: &str) -> Result<Vec<String>> {
265 let pattern = pathspec.strip_prefix(GLOB_PREFIX).unwrap_or(pathspec);
266 let matcher = glob::Glob::new(pattern);
267 let mut out = Vec::new();
268 walk(root, root, &matcher, &self.excluded, &mut out)?;
269 crate::order::canonical(&mut out);
270 Ok(out)
271 }
272}
273
274fn walk(
275 root: &Path,
276 dir: &Path,
277 matcher: &glob::Glob,
278 excluded: &[String],
279 out: &mut Vec<String>,
280) -> Result<()> {
281 let entries = std::fs::read_dir(dir).map_err(|e| Error::io(dir, e))?;
282 for entry in entries {
283 let entry = entry.map_err(|e| Error::io(dir, e))?;
284 let path = entry.path();
285 let Ok(rel) = path.strip_prefix(root) else {
286 continue;
287 };
288 let Some(rel) = rel.to_str() else { continue };
289 let rel = rel.replace('\\', "/");
290 if excluded
291 .iter()
292 .any(|x| rel == *x || rel.starts_with(&format!("{x}/")))
293 {
294 continue;
295 }
296 let meta = std::fs::symlink_metadata(&path).map_err(|e| Error::io(&path, e))?;
297 if meta.file_type().is_symlink() {
298 continue;
299 }
300 if meta.is_dir() {
301 if matcher.could_match_under(&rel) {
302 walk(root, &path, matcher, excluded, out)?;
303 }
304 } else if meta.is_file() && matcher.matches(&rel) {
305 out.push(rel);
306 }
307 }
308 Ok(())
309}
310
311/// Run every class of `sources` through `source`, in declared class order, each class's
312/// files sorted by path. A file two classes both match is kept under the first and reported.
313///
314/// One precedence rule stands above class order, and the manifest states it rather than
315/// this function: a file name listed under `context.documents` carries the context
316/// contract *wherever it appears* under the layer, so it belongs to the class that
317/// produces context documents and to no other. Without it a directory's contract would be
318/// discovered only where some unrelated class's pathspec happened to reach — `.ai/repo/rules/README.md`
319/// through the rule class, `.ai/repo/ci/README.md` through nothing at all — which is
320/// discovery by accident, and it left the root of the hierarchy invisible to every
321/// projection. The rule applies only when a class declares the kind, so a layer that
322/// declares no context class is read exactly as before.
323pub fn discover(
324 repo: &Repository,
325 sources: &Sources,
326 source: &dyn DiscoverySource,
327) -> Result<(Vec<DiscoveredFile>, Vec<Diagnostic>)> {
328 crate::perf::Counters::bump(&crate::perf::COUNTERS.repository_scans);
329 let mut files = Vec::new();
330 let mut diagnostics = Vec::new();
331 let mut claimed = std::collections::BTreeMap::<String, String>::new();
332 let local = repo.local_path();
333 // The manifest's context conventions, and the class that produces them. Both must be
334 // present for the precedence rule to mean anything.
335 let context_class = sources
336 .sources
337 .iter()
338 .find(|c| c.kind == CONTEXT_KIND)
339 .map(|c| c.id.clone());
340 let context_names: Vec<&str> = match (&context_class, repo.manifest().context.as_ref()) {
341 (Some(_), Some(conventions)) => conventions.documents.iter().map(String::as_str).collect(),
342 _ => Vec::new(),
343 };
344 let is_context_document = |rel: &str| -> bool {
345 if context_names.is_empty() || !rel.starts_with(".ai/") {
346 return false;
347 }
348 let name = rel.rsplit('/').next().unwrap_or(rel);
349 context_names.contains(&name)
350 };
351 for class in &sources.sources {
352 let mut paths = source.enumerate(repo.root(), &class.pathspec)?;
353 paths.sort();
354 paths.dedup();
355 let mut count = 0;
356 for rel in paths {
357 if rel == local || rel.starts_with(&format!("{local}/")) {
358 continue;
359 }
360 // The manifest says this name is a context document wherever it appears, so
361 // no other class may claim it. This is precedence, not a collision: reporting
362 // it as claimed_twice would name the layer's own contract a conflict.
363 if is_context_document(&rel) && context_class.as_deref() != Some(class.id.as_str()) {
364 continue;
365 }
366 if let Some(first) = claimed.get(&rel) {
367 diagnostics.push(Diagnostic::warning(
368 "claimed_twice",
369 Some(rel.clone()),
370 format!(
371 "matched by class '{}' after class '{first}'; kept under '{first}'",
372 class.id
373 ),
374 ));
375 continue;
376 }
377 claimed.insert(rel.clone(), class.id.clone());
378 files.push(DiscoveredFile {
379 class: class.id.clone(),
380 kind: class.kind.clone(),
381 rel_path: rel,
382 });
383 count += 1;
384 }
385 if class.required && count == 0 {
386 diagnostics.push(Diagnostic::error(
387 "required_source_empty",
388 None,
389 format!(
390 "required class '{}' ({}) discovered nothing",
391 class.id, class.pathspec
392 ),
393 ));
394 }
395 tracing::debug!(class = %class.id, kind = %class.kind, files = count, "discovered");
396 }
397 Ok((files, diagnostics))
398}
399
400#[cfg(test)]
401mod tests {
402 use super::*;
403
404 #[test]
405 fn sources_parse_and_refuse_bad_shapes() {
406 let ok = "version: 1\nsources:\n - id: rule\n kind: rule\n discovery: vcs\n pathspec: ':(glob).ai/repo/rules/**/*.md'\n required: true\n";
407 let s = Sources::parse(Path::new("s"), ok).unwrap();
408 assert_eq!(s.sources[0].kind, "rule");
409 let dup = format!("{ok} - id: rule\n kind: rule\n discovery: vcs\n pathspec: ':(glob)x'\n required: false\n");
410 assert!(matches!(
411 Sources::parse(Path::new("s"), &dup),
412 Err(Error::InvalidSources { .. })
413 ));
414 let v2 = ok.replace("version: 1", "version: 2");
415 assert!(matches!(
416 Sources::parse(Path::new("s"), &v2),
417 Err(Error::InvalidSources { .. })
418 ));
419 let noglob = ok.replace(":(glob)", "");
420 assert!(matches!(
421 Sources::parse(Path::new("s"), &noglob),
422 Err(Error::InvalidSources { .. })
423 ));
424 let unknown = format!("{ok} colour: red\n");
425 assert!(matches!(
426 Sources::parse(Path::new("s"), &unknown),
427 Err(Error::InvalidSources { .. })
428 ));
429 }
430}