1use std::collections::BTreeMap;
10use std::path::{Path, PathBuf};
11
12use schemars::JsonSchema;
13use serde::{Deserialize, Serialize};
14use serde_json::{Map, Value};
15
16use crate::discovery::{self, DiscoveredFile, DiscoverySource, Sources};
17use crate::error::Result;
18use crate::git::GitState;
19use crate::metadata::{frontmatter, yaml, Format, FrontMatterRule, KindSchema, KindSpec};
20use crate::model::{uri_for, Diagnostic, Object, Provenance, Severity};
21use crate::repository::Repository;
22use crate::scope::{Scope, Scoped, Verdict};
23
24pub const MAX_FILE_BYTES: u64 = 4 * 1024 * 1024;
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
29#[serde(rename_all = "lowercase")]
30pub enum State {
31 Ok,
33 Degraded,
35}
36
37pub const STALE_AFTER: std::time::Duration = std::time::Duration::from_secs(120);
45
46#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
65pub struct AnswerObservation {
66 pub observed_at: String,
69 pub source: String,
71 #[serde(default, skip_serializing_if = "Option::is_none")]
74 pub stale_after: Option<String>,
75}
76
77impl AnswerObservation {
78 pub fn taken(
87 source: &str,
88 at: std::time::SystemTime,
89 window: Option<std::time::Duration>,
90 ) -> Self {
91 AnswerObservation {
92 observed_at: crate::peers::rfc3339(at),
93 source: source.to_string(),
94 stale_after: window.map(|w| crate::peers::rfc3339(at + w)),
95 }
96 }
97
98 pub fn is_stale_at(&self, now: std::time::SystemTime) -> bool {
108 self.stale_after
109 .as_deref()
110 .is_some_and(|limit| crate::peers::rfc3339(now).as_str() > limit)
111 }
112}
113
114#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
132pub struct RepositoryObservations {
133 pub index: AnswerObservation,
135 pub git: AnswerObservation,
137}
138
139#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
141pub struct RepositoryInfo {
142 pub root: String,
144 pub layer_schema: String,
146 pub sections: BTreeMap<String, String>,
148 pub git: GitState,
150 pub discovery: String,
152 pub source_classes: Vec<(String, String)>,
154 pub kind_sources: Vec<String>,
157 pub scope_origin: crate::scope::Origin,
159 pub scope_path: String,
162 #[serde(default)]
165 pub observed: RepositoryObservations,
166}
167
168#[derive(Debug)]
169pub struct Index {
171 pub repository: RepositoryInfo,
173 pub objects: Vec<Object>,
175 pub diagnostics: Vec<Diagnostic>,
177 pub state: State,
179 pub fingerprint: String,
182 pub scoped: Scoped,
184 pub distribution: Option<crate::distribution::Model>,
189 pub providers: crate::share::ProviderDeclarations,
195 pub share: Option<PathBuf>,
207}
208
209impl Index {
210 pub fn build(
213 repo: &Repository,
214 sources: &Sources,
215 schema: &KindSchema,
216 source: &dyn DiscoverySource,
217 git: GitState,
218 scope: Scope,
219 ) -> Result<Self> {
220 let started = std::time::SystemTime::now();
221 let _phase = crate::perf::phase(crate::perf::Phase::IndexBuild);
222 crate::perf::Counters::bump(&crate::perf::COUNTERS.index_builds);
223 let (files, mut diagnostics) = discovery::discover(repo, sources, source)?;
224 let (tally, mut scope_diagnostics) = scope.tally_repository(repo, source)?;
225 diagnostics.append(&mut scope_diagnostics);
226 let mut objects = Vec::with_capacity(files.len());
227 for file in &files {
228 let judged = scope.classify(repo.root(), &file.rel_path);
232 if judged.verdict == Verdict::Out {
233 diagnostics.push(Diagnostic::warning(
234 "out_of_scope",
235 Some(file.rel_path.clone()),
236 format!(
237 "discovered by class '{}' but out of the repository scope ({}{}); not read",
238 file.class,
239 judged.reason.map(|r| r.as_str()).unwrap_or("out"),
240 judged
241 .rule
242 .as_deref()
243 .map(|r| format!(": {r}"))
244 .unwrap_or_default()
245 ),
246 ));
247 continue;
248 }
249 match read_objects(repo, schema, file) {
250 Ok((mut read, mut member_diagnostics)) => {
251 objects.append(&mut read);
252 diagnostics.append(&mut member_diagnostics);
253 }
254 Err(d) => diagnostics.push(d),
255 }
256 }
257 dedupe(&mut objects, &mut diagnostics);
258 objects.sort_by(|a, b| a.uri.cmp(&b.uri));
263 let state = if diagnostics.iter().any(|d| d.severity == Severity::Error) {
264 State::Degraded
265 } else {
266 State::Ok
267 };
268 let repository = RepositoryInfo {
269 root: repo.root().display().to_string(),
270 layer_schema: repo.manifest().schema.clone(),
271 sections: repo
272 .manifest()
273 .sections
274 .keys()
275 .filter_map(|k| repo.section_path(k).map(|p| (k.clone(), p)))
276 .collect(),
277 git,
278 discovery: source.name().to_string(),
279 source_classes: sources
280 .sources
281 .iter()
282 .map(|c| (c.id.clone(), c.kind.clone()))
283 .collect(),
284 kind_sources: schema.sources().to_vec(),
285 scope_origin: scope.origin(),
286 scope_path: scope.path().to_string(),
287 observed: RepositoryObservations {
290 index: AnswerObservation::taken("index build", started, Some(STALE_AFTER)),
291 git: AnswerObservation::taken("git read", started, Some(STALE_AFTER)),
292 },
293 };
294 tracing::info!(
295 objects = objects.len(),
296 diagnostics = diagnostics.len(),
297 state = ?state,
298 "index built"
299 );
300 let fingerprint = {
301 use sha2::{Digest, Sha256};
302 let mut h = Sha256::new();
303 for o in &objects {
304 h.update(o.provenance.path.as_bytes());
305 h.update(b"\0");
306 h.update(o.content.as_bytes());
307 h.update(b"\n");
308 }
309 format!("{:x}", h.finalize())
310 };
311 Ok(Index {
312 repository,
313 objects,
314 diagnostics,
315 state,
316 fingerprint,
317 scoped: Scoped { scope, tally },
318 distribution: None,
319 providers: Default::default(),
320 share: None,
321 })
322 }
323
324 pub fn get(&self, uri: &str) -> Option<&Object> {
326 self.objects
327 .binary_search_by(|o| o.uri.as_str().cmp(uri))
328 .ok()
329 .map(|i| &self.objects[i])
330 }
331
332 pub fn errors(&self) -> usize {
334 self.diagnostics
335 .iter()
336 .filter(|d| d.severity == Severity::Error)
337 .count()
338 }
339
340 pub fn kinds(&self) -> BTreeMap<&str, usize> {
342 let mut m = BTreeMap::new();
343 for o in &self.objects {
344 *m.entry(o.kind.as_str()).or_insert(0) += 1;
345 }
346 m
347 }
348}
349
350fn dedupe(objects: &mut Vec<Object>, diagnostics: &mut Vec<Diagnostic>) {
352 let mut by_uri: BTreeMap<String, Vec<String>> = BTreeMap::new();
353 for o in objects.iter() {
354 by_uri
355 .entry(o.uri.clone())
356 .or_default()
357 .push(o.provenance.path.clone());
358 }
359 let duplicates: BTreeMap<&String, &Vec<String>> =
360 by_uri.iter().filter(|(_, paths)| paths.len() > 1).collect();
361 if duplicates.is_empty() {
362 return;
363 }
364 for (uri, paths) in &duplicates {
365 for path in paths.iter() {
366 diagnostics.push(Diagnostic::error(
367 "duplicate_identity",
368 Some(path.clone()),
369 format!(
370 "{uri} is claimed by {}; every claimant is excluded",
371 paths.join(" and ")
372 ),
373 ));
374 }
375 }
376 objects.retain(|o| !duplicates.contains_key(&o.uri));
377}
378
379fn read_objects(
383 repo: &Repository,
384 schema: &KindSchema,
385 file: &DiscoveredFile,
386) -> std::result::Result<(Vec<Object>, Vec<Diagnostic>), Diagnostic> {
387 let rel = file.rel_path.as_str();
388 let d = |code, msg: String| Diagnostic::error(code, Some(rel.to_string()), msg);
389 let Some(mut spec) = schema.kind(&file.kind) else {
390 return Err(d(
391 "unknown_kind",
392 format!(
393 "class '{}' declares kind '{}', which this executable does not read",
394 file.class, file.kind
395 ),
396 ));
397 };
398 let mut kind = file.kind.as_str();
399 let abs = repo.root().join(rel);
400 let content = read_text(&abs).map_err(|(code, msg)| d(code, msg))?;
401 let bytes = content.len() as u64;
402 let directory = match rel.rsplit_once('/') {
403 Some((dir, _)) => dir.to_string(),
404 None => ".".to_string(),
405 };
406 let provenance = |member: Option<String>| Provenance {
407 path: rel.to_string(),
408 directory: directory.clone(),
409 source_class: file.class.clone(),
410 section: repo.section_of(rel).map(str::to_string),
411 bytes,
412 member,
413 };
414
415 let (metadata, body, media_type): (Map<String, Value>, String, &'static str) = match spec.format
416 {
417 Format::Text => (Map::new(), content.clone(), "text/plain"),
418 Format::Yaml => {
419 let map = yaml::parse_mapping(&content).map_err(|e| d("malformed_yaml", e))?;
420 (map, content.clone(), "application/yaml")
421 }
422 Format::Markdown => {
423 let split = frontmatter::split(&content)
424 .map_err(|e| d("malformed_front_matter", e.to_string()))?;
425 let map = match split.front {
426 Some(front) => {
427 frontmatter::parse(front).map_err(|e| d("malformed_front_matter", e))?
428 }
429 None => Map::new(),
430 };
431 if split.front.is_none() {
432 if let Some(fallback) = spec.without_front_matter.as_deref() {
433 if let Some((name, f)) = schema.kinds().find(|(k, _)| k.as_str() == fallback) {
435 spec = f;
436 kind = name.as_str();
437 }
438 }
439 } else if let Some(declared) = map.get("kind").and_then(Value::as_str) {
440 if declared != kind && schema.is_declared_kind(declared) {
443 if let Some((name, f)) = schema.kinds().find(|(k, _)| k.as_str() == declared) {
444 spec = f;
445 kind = name.as_str();
446 }
447 }
448 }
449 if split.front.is_none() && spec.front_matter == FrontMatterRule::Required {
450 return Err(d(
451 "missing_front_matter",
452 "no front matter, and the kind requires it".into(),
453 ));
454 }
455 (map, split.body.to_string(), "text/markdown")
456 }
457 };
458
459 if let Some(conventions) = &repo.manifest().context {
462 let name = rel.rsplit('/').next().unwrap_or(rel);
463 let under_layer = rel.starts_with(".ai/");
464 if under_layer && conventions.documents.iter().any(|d| d == name) && kind != "context" {
465 return Err(d(
466 "missing_context_contract",
467 format!("the manifest lists {name} among the context documents, and this one declares no schema: context/v1"),
468 ));
469 }
470 }
471
472 if let Some(sv) = &spec.schema_version {
473 check_version(sv, &metadata).map_err(|(code, msg)| d(code, msg))?;
474 }
475
476 if let Some(list) = &spec.members {
478 let Some(items) = metadata.get(list).and_then(Value::as_array) else {
479 return Err(d(
480 "missing_field",
481 format!("no list '{list}' holding the members"),
482 ));
483 };
484 let mut objects = Vec::new();
485 let mut diagnostics = Vec::new();
486 for (i, item) in items.iter().enumerate() {
487 let member = format!("{list}.{i}");
488 let Value::Object(item) = item else {
489 diagnostics.push(d(
490 "malformed_yaml",
491 format!("member {member} is not a mapping"),
492 ));
493 continue;
494 };
495 match object_of(
496 spec,
497 kind,
498 item,
499 "",
500 &member,
501 provenance(Some(member.clone())),
502 ) {
503 Ok(o) => objects.push(o),
504 Err(msg) => diagnostics.push(d("missing_field", format!("member {member}: {msg}"))),
505 }
506 }
507 return Ok((objects, diagnostics));
508 }
509
510 if let Some(sch) = schema.schema_for(spec) {
511 conforms(sch, &Value::Object(metadata.clone()), &d, "")?;
512 }
513 if let Some(declared) = metadata.get("kind").and_then(Value::as_str) {
514 if declared != kind {
515 return Err(d(
516 "kind_mismatch",
517 format!(
518 "declares kind '{declared}' but was discovered by class '{}' as '{kind}'",
519 file.class
520 ),
521 ));
522 }
523 }
524 let object = object_of(spec, kind, &metadata, &body, rel, provenance(None))
525 .map_err(|msg| d("missing_field", msg))?;
526 Ok((
527 vec![Object {
528 content,
529 media_type,
530 ..object
531 }],
532 Vec::new(),
533 ))
534}
535
536fn conforms(
540 sch: &crate::metadata::Schema,
541 instance: &Value,
542 d: &dyn Fn(&'static str, String) -> Diagnostic,
543 prefix: &str,
544) -> std::result::Result<(), Diagnostic> {
545 let violations = sch.validate(instance);
546 if violations.is_empty() {
547 return Ok(());
548 }
549 let unknown: Vec<String> = violations
550 .iter()
551 .flat_map(|v| v.unknown_keys.clone())
552 .collect();
553 if !unknown.is_empty() {
554 return Err(d(
555 "unknown_key",
556 format!(
557 "{prefix}key(s) not in schema '{}': {}",
558 sch.name,
559 unknown.join(", ")
560 ),
561 ));
562 }
563 let list: Vec<String> = violations
564 .iter()
565 .map(|v| {
566 if v.path.is_empty() {
567 v.message.clone()
568 } else {
569 format!("{}: {}", v.path, v.message)
570 }
571 })
572 .collect();
573 Err(d(
574 "schema_violation",
575 format!("{prefix}schema '{}': {}", sch.name, list.join("; ")),
576 ))
577}
578
579fn check_version(
580 sv: &crate::metadata::SchemaVersion,
581 metadata: &Map<String, Value>,
582) -> std::result::Result<(), (&'static str, String)> {
583 match metadata.get(&sv.field) {
584 Some(v) if sv.supported.contains(v) => Ok(()),
585 Some(v) => Err((
586 "unsupported_version",
587 format!(
588 "{} {v} is not among the supported {}",
589 sv.field,
590 serde_json::to_string(&sv.supported).unwrap_or_default()
591 ),
592 )),
593 None => Err(("missing_field", format!("no '{}'", sv.field))),
594 }
595}
596
597fn object_of(
600 spec: &KindSpec,
601 kind: &str,
602 metadata: &Map<String, Value>,
603 body: &str,
604 fallback_identity: &str,
605 provenance: Provenance,
606) -> std::result::Result<Object, String> {
607 let identity = identity_of(spec, metadata, fallback_identity)?;
608 let title = title_of(spec, metadata, body);
609 let description = spec
610 .description
611 .as_deref()
612 .and_then(|f| metadata.get(f))
613 .and_then(yaml::scalar_string);
614 let is_member = provenance.member.is_some();
615 let text = if is_member {
616 serde_json::to_string_pretty(&Value::Object(metadata.clone())).unwrap_or_default()
617 } else {
618 String::new()
619 };
620 Ok(Object {
621 uri: uri_for(kind, &identity),
622 kind: kind.to_string(),
623 identity,
624 title,
625 description,
626 metadata: Value::Object(metadata.clone()),
627 body: if is_member {
628 text.clone()
629 } else {
630 body.to_string()
631 },
632 content: text,
633 media_type: if is_member {
634 "application/json"
635 } else {
636 "text/plain"
637 },
638 provenance,
639 })
640}
641
642fn identity_of(
643 spec: &KindSpec,
644 metadata: &Map<String, Value>,
645 rel: &str,
646) -> std::result::Result<String, String> {
647 if spec.identity.is_empty() {
648 return Ok(rel.to_string());
649 }
650 let mut parts = Vec::with_capacity(spec.identity.len());
651 for field in &spec.identity {
652 let value = metadata
653 .get(field)
654 .and_then(yaml::scalar_string)
655 .filter(|s| !s.is_empty())
656 .ok_or_else(|| format!("identity field '{field}' is absent or empty"))?;
657 if value.contains('/') || value.contains('@') || value.contains(char::is_whitespace) {
658 return Err(format!(
659 "identity field '{field}' carries '/', '@' or whitespace: '{value}'"
660 ));
661 }
662 parts.push(value);
663 }
664 Ok(parts.join("@"))
665}
666
667fn title_of(spec: &KindSpec, metadata: &Map<String, Value>, body: &str) -> Option<String> {
668 match spec.title.as_deref()? {
669 "@heading" => body
670 .lines()
671 .find_map(|l| l.strip_prefix("# "))
672 .map(|t| t.trim().to_string())
673 .filter(|t| !t.is_empty()),
674 field => metadata.get(field).and_then(yaml::scalar_string),
675 }
676}
677
678fn read_text(abs: &Path) -> std::result::Result<String, (&'static str, String)> {
680 let meta = std::fs::symlink_metadata(abs).map_err(|e| ("unreadable", e.to_string()))?;
681 if meta.file_type().is_symlink() {
682 return Err((
683 "symlink",
684 "a symlink is not read; sources are regular files".into(),
685 ));
686 }
687 if !meta.is_file() {
688 return Err(("not_a_file", "not a regular file".into()));
689 }
690 if meta.len() > MAX_FILE_BYTES {
691 return Err((
692 "oversized",
693 format!("{} bytes, over the {MAX_FILE_BYTES} byte limit", meta.len()),
694 ));
695 }
696 let bytes = std::fs::read(abs).map_err(|e| ("unreadable", e.to_string()))?;
697 String::from_utf8(bytes).map_err(|e| ("invalid_utf8", e.to_string()))
698}