Skip to main content

majordomus_cli/mesh/
cooperation.rs

1//! Cooperation: the runtime that turns discovered, trusted runtimes of one repository into
2//! linked peers and keeps their journal converged. It owns the one peer-link table, the
3//! one journal of this runtime, the link handlers the HTTP surface answers, the dialers,
4//! the heartbeat and the expiry — and the operations (sessions, claims, handovers,
5//! reviews) every surface performs through it.
6//!
7//! The sequence one link goes through, and where each step is decided:
8//!
9//! ```text
10//! candidate  a trusted registry record of the same repository, or a declared seed
11//!    ↓
12//! dial       hello → welcome  (link::)  signature, freshness, nonce, protocol,
13//!                                        repository, self, trust — both directions
14//!    ↓
15//! connected  sync every heartbeat: marks + missing events, both ways, signed
16//!    ↓        a failed round: degraded → unreachable (backoff, capped at the expiry)
17//! expired    no exchange within the expiry: the link is dropped, the runtime's
18//!    ↓        streams stop beating everywhere, its claims stop excluding
19//! reconnect  the dialer says hello again; an unknown link id or a new instance is a
20//!             re-handshake, never a second peer — the table is keyed by runtime
21//! ```
22//!
23//! StreamLiveness has two layers and they are deliberately separate. A *link* is this runtime's
24//! connection to one peer (connected, degraded, unreachable, expired). A *stream* is
25//! alive while its beat keeps rising anywhere in the mesh — a runtime linked only through
26//! a relay is still alive, and a runtime that crashed stops beating for everyone at once.
27//! Claims follow streams, not links.
28//!
29//! The whole of it, two runtimes in one process. Only the socket is replaced: the
30//! handshake, the signatures, the admission and the fold are the ones a LAN takes.
31//!
32//! ```
33//! use std::collections::BTreeMap;
34//! use std::sync::{Arc, Mutex, Weak};
35//!
36//! use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
37//! use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
38//! use majordomus_cli::mesh::identity::NodeIdentity;
39//! use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
40//! use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
41//! use majordomus_cli::mesh::registry::MeshRegistry;
42//! use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
43//! use majordomus_cli::mesh::trust::TrustPolicy;
44//!
45//! #[derive(Default)]
46//! struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
47//!
48//! impl LinkTransport for Net {
49//!     fn post(&self, endpoint: &str, path: &str, m: &Signed) -> Result<LinkReply, String> {
50//!         let peer = (self.0.lock().unwrap().get(endpoint).and_then(Weak::upgrade))
51//!             .ok_or_else(|| format!("{endpoint}: nobody there"))?;
52//!         Ok(if path == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) })
53//!     }
54//! }
55//!
56//! let net = Arc::new(Net::default());
57//! let runtime = |endpoint: &str| {
58//!     let c = Cooperation::new(CooperationSetup {
59//!         identity: Arc::new(NodeIdentity::ephemeral().unwrap()),
60//!         runtime: runtime_id(endpoint),
61//!         repository: of_root_commits(&["root".into()]),
62//!         endpoints: vec![endpoint.into()],
63//!         version: "doc".into(),
64//!         config: CooperationConfig::default(),
65//!         trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] },
66//!         journal_path: None,
67//!         registry: Arc::new(MeshRegistry::new()),
68//!         transport: Arc::clone(&net) as Arc<dyn LinkTransport>,
69//!         board: None,
70//!         checkout: CheckoutFacts::default(),
71//!     })
72//!     .unwrap();
73//!     net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c));
74//!     c
75//! };
76//!
77//! let (a, b) = (runtime("a:1"), runtime("b:1"));
78//! let key = a.dial(&["b:1".into()]).expect("B welcomes A");
79//! assert_eq!(key, b.runtime_key(), "the link is keyed by the peer's runtime");
80//!
81//! a.claim(&SessionInfo::named("s1", "cli"), vec!["apps".into()], None,
82//!     ClaimMode::Exclusive, None).unwrap();
83//! for _ in 0..2 {
84//!     a.journal().beat_own();
85//!     a.sync_with(&key).unwrap();
86//! }
87//! assert_eq!(b.state().claims.len(), 1, "the claim crossed the link");
88//! assert_eq!(a.state().digest, b.state().digest, "one state, two runtimes");
89//! ```
90
91use std::collections::BTreeMap;
92use std::path::PathBuf;
93use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
94use std::sync::{Arc, Mutex};
95use std::time::{Duration, Instant};
96
97use schemars::JsonSchema;
98use serde::{Deserialize, Serialize};
99
100use super::config::{CooperationConfig, TrustConfig};
101use super::identity::{node_id_of_key, NodeIdentity};
102use super::journal::{
103    ClaimMode, Durability, EventBody, HandoverBody, Journal, JournalTallies, Marks, MeshEvent,
104    Rejection, SessionInfo, StreamId, StreamLiveness, HANDOVER_RETENTION,
105};
106use super::link::{
107    fresh_token, negotiate, sign, verify_signed, Domain, Hello, LinkRefusal, LinkReply,
108    LinkTransport, RefusalCode, RuntimeCard, Signed, SyncAnswer, SyncRequest, Welcome, FEATURES,
109    HELLO_PATH, LINK_PROTOCOL_MAX, LINK_PROTOCOL_MIN, MAX_LINK_MESSAGE, MAX_LINK_SKEW_SECONDS,
110    SYNC_EVENT_BUDGET, SYNC_PATH,
111};
112use super::registry::{MeshRegistry, NodeRecord, Presence};
113use super::repository::MeshRepositoryIdentity;
114use super::state::{admission_conflicts, fold, ClaimView, CooperationState, HandoverView};
115use super::trust::{self, TrustState};
116
117/// The most peers one runtime links to.
118pub const MAX_PEERS: usize = 256;
119
120/// How long an expired peer stays listed before it leaves the table.
121pub const PEER_RETENTION: Duration = Duration::from_secs(15 * 60);
122
123/// The most hello nonces remembered for replay refusal.
124const NONCE_CACHE: usize = 4096;
125
126/// The stop check's slice while sleeping.
127const SLICE: Duration = Duration::from_millis(100);
128
129/// Everything one runtime needs before it can link to another, gathered in one place so
130/// that the runtime never reaches for it later. Each field is a decision somebody else
131/// already made — the declaration's policy, the checkout's identity, this process's key,
132/// the socket to speak through — and passing them in is what lets a test replace any of
133/// them without a fixture: a fake transport, an ephemeral key, a journal that never
134/// touches the disk.
135///
136/// ```
137/// use std::sync::Arc;
138///
139/// use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
140/// use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
141/// use majordomus_cli::mesh::identity::NodeIdentity;
142/// use majordomus_cli::mesh::link::HttpTransport;
143/// use majordomus_cli::mesh::registry::MeshRegistry;
144/// use majordomus_cli::mesh::repository::of_root_commits;
145///
146/// let setup = CooperationSetup {
147///     identity: Arc::new(NodeIdentity::ephemeral().unwrap()),
148///     runtime: "0000000000000001".into(),
149///     repository: of_root_commits(&["root".into()]),
150///     endpoints: vec!["127.0.0.1:9".into()],
151///     version: "doc".into(),
152///     config: CooperationConfig::default(),
153///     trust: TrustConfig::default(),
154///     journal_path: None,
155///     registry: Arc::new(MeshRegistry::new()),
156///     transport: Arc::new(HttpTransport),
157///     board: None,
158///     checkout: CheckoutFacts::default(),
159/// };
160///
161/// let cooperation = Cooperation::new(setup).unwrap();
162/// assert!(cooperation.runtime_key().ends_with("-0000000000000001"));
163/// ```
164pub struct CooperationSetup {
165    /// This node's identity.
166    pub identity: Arc<NodeIdentity>,
167    /// This runtime's slot, 16 hex.
168    pub runtime: String,
169    /// The repository's mesh identity.
170    pub repository: MeshRepositoryIdentity,
171    /// Where this runtime answers.
172    pub endpoints: Vec<String>,
173    /// The executable version.
174    pub version: String,
175    /// The declaration's cooperation section.
176    pub config: CooperationConfig,
177    /// The declaration's trust section.
178    pub trust: TrustConfig,
179    /// Where the journal persists, when anywhere.
180    pub journal_path: Option<PathBuf>,
181    /// The discovery registry candidates come from.
182    pub registry: Arc<MeshRegistry>,
183    /// How link messages travel.
184    pub transport: Arc<dyn LinkTransport>,
185    /// The peer board of this process, projected into the journal every heartbeat: each
186    /// attached session becomes a mesh session and each of its announcements an advisory
187    /// claim, so a board announcement is visible on every linked runtime.
188    pub board: Option<Arc<crate::peers::PeerBoard>>,
189    /// Facts about this checkout the projected sessions carry.
190    pub checkout: CheckoutFacts,
191}
192
193/// What the projected sessions say about the checkout they work in. Computed once at
194/// start — branch and head are refreshed when a board session changes, never on a
195/// heartbeat, so liveness never rescans the repository.
196///
197/// Both fields are optional because a runtime need not be in a checkout at all, and a
198/// runtime that is says so once: the root is kept rather than the branch, so that a
199/// session announced after a checkout moved carries where it is now and not where it was
200/// when the process started.
201///
202/// ```
203/// use majordomus_cli::mesh::cooperation::CheckoutFacts;
204///
205/// let unknown = CheckoutFacts::default();
206/// assert!(unknown.root.is_none(), "a runtime outside a checkout claims no branch");
207///
208/// let here = CheckoutFacts { id: Some("c0ffee".into()), root: Some(".".into()) };
209/// assert_eq!(here.id.as_deref(), Some("c0ffee"));
210/// ```
211#[derive(Debug, Clone, Default)]
212pub struct CheckoutFacts {
213    /// The checkout id (a digest).
214    pub id: Option<String>,
215    /// The checkout root, for reading the branch and the head when a session changes.
216    pub root: Option<PathBuf>,
217}
218
219impl CheckoutFacts {
220    fn git(&self, args: &[&str]) -> Option<String> {
221        let root = self.root.as_ref()?;
222        let out = std::process::Command::new("git")
223            .arg("-C")
224            .arg(root)
225            .args(args)
226            .output()
227            .ok()?;
228        out.status
229            .success()
230            .then(|| String::from_utf8_lossy(&out.stdout).trim().to_string())
231            .filter(|s| !s.is_empty())
232    }
233}
234
235#[derive(Default)]
236struct Projected {
237    info: SessionInfo,
238    claims: BTreeMap<String, (String, Vec<String>, String)>,
239}
240
241/// A link's standing, from this runtime's side. It is a statement about a connection and
242/// never about the peer: a runtime reachable only through a third party is `Expired` here
243/// while its streams beat on everywhere, which is why claims follow streams and not this.
244///
245/// The variants are ordered by how far the link has fallen, so the worst link in a table
246/// is the maximum and a reader never has to rank them by hand.
247///
248/// ```
249/// use majordomus_cli::mesh::cooperation::LinkState;
250///
251/// let table = [LinkState::Connected, LinkState::Unreachable, LinkState::Degraded];
252/// assert_eq!(table.iter().max(), Some(&LinkState::Unreachable), "the worst link shows");
253/// assert_eq!(serde_json::to_value(LinkState::Connected).unwrap(), "connected");
254/// ```
255#[derive(
256    Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema,
257)]
258#[serde(rename_all = "snake_case")]
259pub enum LinkState {
260    /// Handshaking, or never yet exchanged.
261    Connecting,
262    /// An exchange within two heartbeats.
263    Connected,
264    /// Rounds are failing; the last exchange is under half the expiry old.
265    Degraded,
266    /// The last exchange is over half the expiry old; the link is about to expire.
267    Unreachable,
268    /// No exchange within the expiry: the link is dropped and will be re-dialed.
269    Expired,
270}
271
272#[derive(Clone)]
273struct Outbound {
274    link: String,
275    endpoint: String,
276    counter: u64,
277    remote_marks: Marks,
278}
279
280#[derive(Clone)]
281struct Inbound {
282    link: String,
283    counter: u64,
284}
285
286struct PeerLink {
287    card: RuntimeCard,
288    trust: TrustState,
289    proto: u32,
290    outbound: Option<Outbound>,
291    inbound: Option<Inbound>,
292    last_exchange: Option<Instant>,
293    linked_at: String,
294    rtt: Option<Duration>,
295    failures: u32,
296    last_error: Option<String>,
297    handshakes: u64,
298    reconnects: u64,
299    restarts: u64,
300    expired: bool,
301}
302
303struct Refused {
304    refusal: LinkRefusal,
305    runtime: Option<String>,
306    direction: &'static str,
307    at: String,
308}
309
310#[derive(Default)]
311struct Table {
312    peers: BTreeMap<String, PeerLink>,
313    inbound_links: BTreeMap<String, String>,
314    nonces: BTreeMap<(String, String), u64>,
315    refused: BTreeMap<String, Refused>,
316    workers: BTreeMap<String, Arc<AtomicBool>>,
317}
318
319#[derive(Default)]
320struct Counters {
321    handshakes_out: AtomicU64,
322    handshakes_in: AtomicU64,
323    dial_failures: AtomicU64,
324    syncs_out: AtomicU64,
325    syncs_failed: AtomicU64,
326    syncs_in: AtomicU64,
327    events_sent: AtomicU64,
328    events_served: AtomicU64,
329    reconnects: AtomicU64,
330    restarts: AtomicU64,
331    peers_expired: AtomicU64,
332    claims_refused: AtomicU64,
333    threads: AtomicU64,
334    refused_in: Mutex<BTreeMap<RefusalCode, u64>>,
335    refused_out: Mutex<BTreeMap<RefusalCode, u64>>,
336}
337
338/// One peer as every surface shows it: who it is, how this runtime reaches it, and what
339/// the link has been through. The counters are carried per peer rather than only in the
340/// totals because the question a person asks of a flapping mesh is which peer is flapping,
341/// and `restarts` answers a different one from `reconnects` — a peer whose process keeps
342/// dying, against a link that keeps dropping under a process that never did.
343///
344/// ```
345/// # use std::collections::BTreeMap;
346/// # use std::sync::{Arc, Mutex, Weak};
347/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
348/// # use majordomus_cli::mesh::cooperation::*;
349/// # use majordomus_cli::mesh::identity::NodeIdentity;
350/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
351/// # use majordomus_cli::mesh::registry::MeshRegistry;
352/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
353/// # use majordomus_cli::mesh::trust::TrustPolicy;
354/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
355/// # impl LinkTransport for Net {
356/// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
357/// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
358/// #         .ok_or_else(|| format!("{e}: nobody there"))?;
359/// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
360/// # let net = Arc::new(Net::default());
361/// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
362/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
363/// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
364/// #     version: "doc".into(), config: CooperationConfig::default(),
365/// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
366/// #     registry: Arc::new(MeshRegistry::new()),
367/// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
368/// #     checkout: CheckoutFacts::default() }).unwrap();
369/// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
370/// # let (a, b) = (runtime("a:1"), runtime("b:1"));
371/// a.dial(&["b:1".into()]).unwrap();
372///
373/// let peer: &PeerView = &a.peers()[0];
374/// assert_eq!(peer.runtime, b.runtime_key(), "the durable identity, not the endpoint");
375/// assert!(peer.outbound, "this runtime dials it");
376/// assert_eq!(peer.handshakes, 1);
377/// assert_eq!(peer.restarts, 0, "the peer's process has not died under the link");
378/// ```
379#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
380pub struct PeerView {
381    /// `<node>-<runtime>`: the peer's durable identity.
382    pub runtime: String,
383    /// The node (machine key digest).
384    pub node: String,
385    /// The current instance.
386    pub instance: String,
387    /// The node's display name.
388    pub name: String,
389    /// The executable version.
390    pub version: String,
391    /// Whether the peer runs on this machine (the same node key).
392    pub local: bool,
393    /// The trust verdict the link was admitted under.
394    pub trust: TrustState,
395    /// The negotiated link protocol.
396    pub protocol: u32,
397    /// The link features the peer carries.
398    pub features: Vec<String>,
399    /// Where the peer answers.
400    pub endpoints: Vec<String>,
401    /// The link's standing.
402    pub state: LinkState,
403    /// Whether this runtime dials the peer.
404    pub outbound: bool,
405    /// Whether the peer dials this runtime.
406    pub inbound: bool,
407    /// The endpoint this runtime dials, when it does.
408    #[serde(default, skip_serializing_if = "Option::is_none")]
409    pub endpoint: Option<String>,
410    /// Milliseconds since the last successful exchange in either direction.
411    #[serde(default, skip_serializing_if = "Option::is_none")]
412    pub last_exchange_ms: Option<u64>,
413    /// The last round trip, milliseconds.
414    #[serde(default, skip_serializing_if = "Option::is_none")]
415    pub rtt_ms: Option<u64>,
416    /// Consecutive failed rounds.
417    pub failures: u32,
418    /// The last failure, in words.
419    #[serde(default, skip_serializing_if = "Option::is_none")]
420    pub last_error: Option<String>,
421    /// Handshakes completed with this peer.
422    pub handshakes: u64,
423    /// Handshakes after an expiry or a dropped link.
424    pub reconnects: u64,
425    /// New instances of the peer seen (its process restarted).
426    pub restarts: u64,
427    /// When the current link was established, RFC 3339.
428    pub linked_at: String,
429}
430
431/// A candidate that was refused, with the rule that refused it. A refusal is kept and
432/// shown rather than logged and dropped, because the commonest mesh failure is a link that
433/// never forms: a person who declared a seed and sees no peer needs the decision that was
434/// made about it — the wrong repository, a key nobody allowed, a clock too far out — and
435/// the direction says which side made it.
436///
437/// ```
438/// # use std::collections::BTreeMap;
439/// # use std::sync::{Arc, Mutex, Weak};
440/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
441/// # use majordomus_cli::mesh::cooperation::*;
442/// # use majordomus_cli::mesh::identity::NodeIdentity;
443/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, RefusalCode, Signed, HELLO_PATH};
444/// # use majordomus_cli::mesh::registry::MeshRegistry;
445/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
446/// # use majordomus_cli::mesh::trust::TrustPolicy;
447/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
448/// # impl LinkTransport for Net {
449/// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
450/// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
451/// #         .ok_or_else(|| format!("{e}: nobody there"))?;
452/// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
453/// # let net = Arc::new(Net::default());
454/// # let runtime = |endpoint: &str, policy: TrustPolicy| {
455/// #   let c = Cooperation::new(CooperationSetup {
456/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
457/// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
458/// #     version: "doc".into(), config: CooperationConfig::default(),
459/// #     trust: TrustConfig { policy, allow: vec![] }, journal_path: None,
460/// #     registry: Arc::new(MeshRegistry::new()),
461/// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
462/// #     checkout: CheckoutFacts::default() }).unwrap();
463/// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
464/// # let a = runtime("a:1", TrustPolicy::Tofu);
465/// # let strict = runtime("s:1", TrustPolicy::DenyUnknown);
466/// a.dial(&["s:1".into()]).expect_err("reachable, and not trusted");
467///
468/// let refused: &RefusedView = &strict.status().refused[0];
469/// assert_eq!(refused.refusal.code, RefusalCode::Untrusted);
470/// assert_eq!(refused.direction, "inbound", "this runtime refused the hello");
471/// assert_eq!(refused.runtime.as_deref(), Some(a.runtime_key()));
472/// ```
473#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
474pub struct RefusedView {
475    /// The endpoint dialed, or `inbound:<runtime>` for a refused hello.
476    pub endpoint: String,
477    /// `outbound` (this runtime refused the peer's welcome, or the peer refused this
478    /// hello) or `inbound` (this runtime refused the peer's hello).
479    pub direction: String,
480    /// The peer's runtime key, when its card said.
481    #[serde(default, skip_serializing_if = "Option::is_none")]
482    pub runtime: Option<String>,
483    /// The refusal.
484    pub refusal: LinkRefusal,
485    /// When, RFC 3339.
486    pub at: String,
487}
488
489/// The link and replication counters since start: what this runtime did, not what it
490/// holds. They are monotonic and never reset, so two readings a minute apart are a rate —
491/// which is the only way to tell a mesh that is working from one that is retrying, since
492/// both show the same peers in the same states.
493///
494/// The counts are split by direction throughout (`handshakes_out` against
495/// `handshakes_in`, `refused_in` against `refused_out`) because a runtime that refuses
496/// everyone and one that everyone refuses need opposite remedies.
497///
498/// ```
499/// # use std::collections::BTreeMap;
500/// # use std::sync::{Arc, Mutex, Weak};
501/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
502/// # use majordomus_cli::mesh::cooperation::*;
503/// # use majordomus_cli::mesh::identity::NodeIdentity;
504/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
505/// # use majordomus_cli::mesh::registry::MeshRegistry;
506/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
507/// # use majordomus_cli::mesh::trust::TrustPolicy;
508/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
509/// # impl LinkTransport for Net {
510/// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
511/// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
512/// #         .ok_or_else(|| format!("{e}: nobody there"))?;
513/// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
514/// # let net = Arc::new(Net::default());
515/// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
516/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
517/// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
518/// #     version: "doc".into(), config: CooperationConfig::default(),
519/// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
520/// #     registry: Arc::new(MeshRegistry::new()),
521/// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
522/// #     checkout: CheckoutFacts::default() }).unwrap();
523/// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
524/// # let (a, b) = (runtime("a:1"), runtime("b:1"));
525/// let key = a.dial(&["b:1".into()]).unwrap();
526/// a.sync_with(&key).unwrap();
527///
528/// let dialer: CooperationCounters = a.status().counters;
529/// let answerer = b.status().counters;
530/// assert_eq!((dialer.handshakes_out, dialer.syncs_out), (1, 1));
531/// assert_eq!((answerer.handshakes_in, answerer.syncs_in), (1, 1), "the other side of it");
532/// assert_eq!(answerer.handshakes_out, 0, "B never dialed anybody");
533/// ```
534#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
535pub struct CooperationCounters {
536    /// Handshakes this runtime completed as the dialer.
537    pub handshakes_out: u64,
538    /// Handshakes this runtime accepted.
539    pub handshakes_in: u64,
540    /// Dials that reached no endpoint.
541    pub dial_failures: u64,
542    /// Sync rounds this runtime completed as the dialer.
543    pub syncs_out: u64,
544    /// Sync rounds that failed in transport.
545    pub syncs_failed: u64,
546    /// Sync rounds this runtime answered.
547    pub syncs_in: u64,
548    /// Events pushed in sync requests.
549    pub events_sent: u64,
550    /// Events served in sync answers.
551    pub events_served: u64,
552    /// Re-handshakes after an expiry or a dropped link.
553    pub reconnects: u64,
554    /// Peer restarts observed.
555    pub restarts: u64,
556    /// Links expired.
557    pub peers_expired: u64,
558    /// Claims refused for a conflict.
559    pub claims_refused: u64,
560    /// Cooperation threads alive now.
561    pub threads: u64,
562    /// Hellos and syncs this runtime refused, by code.
563    pub refused_in: BTreeMap<RefusalCode, u64>,
564    /// Refusals this runtime received or applied to a welcome, by code.
565    pub refused_out: BTreeMap<RefusalCode, u64>,
566}
567
568/// The whole cooperation runtime, as every surface reports it: the command line, the HTTP
569/// surface and MCP all render this one value, so that a person and an agent asking the same
570/// question of the same runtime are never told different things.
571///
572/// It answers in one reading the three questions a mesh raises — is this runtime
573/// cooperating at all, what does it speak, and who is it linked to — and it stays
574/// answerable when the answer is no: an inactive runtime still reports its protocol range
575/// and the reason, rather than an absence the caller has to interpret.
576///
577/// ```
578/// # use std::sync::Arc;
579/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
580/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
581/// # use majordomus_cli::mesh::identity::NodeIdentity;
582/// # use majordomus_cli::mesh::link::HttpTransport;
583/// # use majordomus_cli::mesh::registry::MeshRegistry;
584/// # use majordomus_cli::mesh::repository::of_root_commits;
585/// # let cooperation = Cooperation::new(CooperationSetup {
586/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
587/// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
588/// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
589/// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
590/// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
591/// # }).unwrap();
592/// use majordomus_cli::mesh::cooperation::CooperationStatus;
593///
594/// let running: CooperationStatus = cooperation.status();
595/// assert_eq!(running.runtime.as_deref(), Some(cooperation.runtime_key()));
596/// assert!(running.peers.is_empty(), "nothing is dialed until something dials");
597///
598/// // and where there is no runtime at all, the same shape says why
599/// let none = CooperationStatus::inactive("the mesh declaration is disabled");
600/// assert_eq!(none.protocol, running.protocol, "still answerable about what it speaks");
601/// ```
602#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
603pub struct CooperationStatus {
604    /// Whether cooperation runs in this process.
605    pub active: bool,
606    /// Why not, when it does not.
607    #[serde(default, skip_serializing_if = "Option::is_none")]
608    pub reason: Option<String>,
609    /// This runtime, `<node>-<runtime>`.
610    #[serde(default, skip_serializing_if = "Option::is_none")]
611    pub runtime: Option<String>,
612    /// This runtime's current stream.
613    #[serde(default, skip_serializing_if = "Option::is_none")]
614    pub stream: Option<StreamId>,
615    /// The repository identity links are matched on.
616    #[serde(default, skip_serializing_if = "Option::is_none")]
617    pub repository: Option<MeshRepositoryIdentity>,
618    /// The link protocol range spoken.
619    pub protocol: (u32, u32),
620    /// The link features carried.
621    pub features: Vec<String>,
622    /// Seconds between heartbeats.
623    pub heartbeat_seconds: u64,
624    /// Seconds of silence before expiry.
625    pub expiry_seconds: u64,
626    /// Where this runtime answers.
627    pub endpoints: Vec<String>,
628    /// The declared seeds.
629    pub seeds: Vec<String>,
630    /// Every linked peer, by runtime key.
631    pub peers: Vec<PeerView>,
632    /// Candidates refused, by endpoint.
633    pub refused: Vec<RefusedView>,
634    /// Counters.
635    pub counters: CooperationCounters,
636    /// The journal's tallies.
637    pub journal: JournalTallies,
638    /// When cooperation started, RFC 3339.
639    #[serde(default, skip_serializing_if = "Option::is_none")]
640    pub started_at: Option<String>,
641}
642
643impl CooperationStatus {
644    /// The status of a runtime where cooperation is not running, and why.
645    ///
646    /// The protocol range and the features are still filled in, because what this
647    /// executable would speak is knowable without a runtime and a person comparing two
648    /// machines needs it precisely when one of them is not cooperating.
649    ///
650    /// ```
651    /// use majordomus_cli::mesh::cooperation::CooperationStatus;
652    ///
653    /// let status = CooperationStatus::inactive("no mesh declaration in this repository");
654    /// assert!(!status.active);
655    /// assert!(status.reason.unwrap().contains("no mesh declaration"), "never a bare false");
656    /// assert!(!status.features.is_empty(), "what it would speak is known without a runtime");
657    /// ```
658    pub fn inactive(reason: &str) -> Self {
659        CooperationStatus {
660            active: false,
661            reason: Some(reason.into()),
662            runtime: None,
663            stream: None,
664            repository: None,
665            protocol: (LINK_PROTOCOL_MIN, LINK_PROTOCOL_MAX),
666            features: FEATURES.iter().map(|f| f.to_string()).collect(),
667            heartbeat_seconds: 0,
668            expiry_seconds: 0,
669            endpoints: Vec::new(),
670            seeds: Vec::new(),
671            peers: Vec::new(),
672            refused: Vec::new(),
673            counters: CooperationCounters::default(),
674            journal: JournalTallies::default(),
675            started_at: None,
676        }
677    }
678}
679
680/// Why a cooperation operation did not happen. Every variant is a decision rather than a
681/// fault: nothing here is a transport failure or a bug, so a caller that sees one has been
682/// told something true about the mesh and should say it to its user rather than retry.
683///
684/// `Conflict` carries the claims it met instead of a message about them, because the
685/// worker being refused needs the other side's scope, session and runtime to go and talk
686/// to whoever holds it.
687///
688/// ```
689/// # use std::sync::Arc;
690/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
691/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
692/// # use majordomus_cli::mesh::identity::NodeIdentity;
693/// # use majordomus_cli::mesh::link::HttpTransport;
694/// # use majordomus_cli::mesh::registry::MeshRegistry;
695/// # use majordomus_cli::mesh::repository::of_root_commits;
696/// # let cooperation = Cooperation::new(CooperationSetup {
697/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
698/// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
699/// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
700/// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
701/// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
702/// # }).unwrap();
703/// use majordomus_cli::mesh::cooperation::CooperationError;
704/// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
705///
706/// let held = SessionInfo::named("s1", "cli");
707/// cooperation.claim(&held, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
708///
709/// let latecomer = SessionInfo::named("s2", "cli");
710/// let refused = cooperation
711///     .claim(&latecomer, vec!["apps/cli".into()], None, ClaimMode::Exclusive, None)
712///     .unwrap_err();
713/// match &refused {
714///     CooperationError::Conflict(claims) => assert_eq!(claims[0].scope, ["apps"]),
715///     other => panic!("expected a conflict, got {other}"),
716/// }
717/// assert!(refused.to_string().contains("apps"), "the message names what it met");
718/// ```
719#[derive(Debug, Clone, thiserror::Error)]
720pub enum CooperationError {
721    /// An exclusive claim meets live exclusive claims of other sessions.
722    #[error("the scope is claimed: {}", .0.iter().map(|c| format!("{} ({})", c.key, c.scope.join(", "))).collect::<Vec<_>>().join("; "))]
723    Conflict(Vec<ClaimView>),
724    /// The operation needs something this runtime wrote, and this was written elsewhere.
725    #[error("{0}")]
726    NotOwn(String),
727    /// Nothing by that name.
728    #[error("{0}")]
729    NotFound(String),
730    /// The peer lacks a feature the operation needs.
731    #[error("{0}")]
732    FeatureUnsupported(String),
733    /// Out of bounds.
734    #[error("{0}")]
735    Invalid(String),
736}
737
738/// One verification check: what was checked, whether it holds, the evidence, and — when
739/// something is wrong or limited — the impact and what to do about it.
740///
741/// The impact and the remedy are carried with the verdict rather than left to whoever
742/// renders it, so that a failing check is actionable wherever it is read — in a terminal,
743/// in a JSON answer, or by an agent that has never seen the mesh before. A check that
744/// holds carries neither, because there is nothing to do.
745///
746/// ```
747/// # use std::sync::Arc;
748/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
749/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
750/// # use majordomus_cli::mesh::identity::NodeIdentity;
751/// # use majordomus_cli::mesh::link::HttpTransport;
752/// # use majordomus_cli::mesh::registry::MeshRegistry;
753/// # use majordomus_cli::mesh::repository::of_root_commits;
754/// # let cooperation = Cooperation::new(CooperationSetup {
755/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
756/// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
757/// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
758/// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
759/// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
760/// # }).unwrap();
761/// use majordomus_cli::mesh::cooperation::MeshVerifyCheck;
762///
763/// let report = cooperation.verify();
764/// let first: &MeshVerifyCheck = &report.checks[0];
765/// assert_eq!(first.check, "cooperation");
766/// assert!(first.detail.contains(cooperation.runtime_key()), "the evidence, not a verdict");
767/// assert!(first.remediation.is_none(), "a check that holds asks for nothing");
768/// ```
769#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
770pub struct MeshVerifyCheck {
771    /// What was checked.
772    pub check: String,
773    /// Whether it holds.
774    pub ok: bool,
775    /// The evidence.
776    pub detail: String,
777    /// What the failure or limitation means for cooperation.
778    #[serde(default, skip_serializing_if = "Option::is_none")]
779    pub impact: Option<String>,
780    /// What to do.
781    #[serde(default, skip_serializing_if = "Option::is_none")]
782    pub remediation: Option<String>,
783}
784
785/// What one sync round, run now rather than remembered, said about one peer. Verification
786/// does not report the link's standing and stop there: it exchanges with the peer and says
787/// whether the exchange worked and whether both sides ended holding the same marks, which
788/// is the difference between a peer that answers and a peer that agrees.
789///
790/// `round_trip` and `converged` are optional because a peer this runtime does not dial is
791/// not asked — an absent answer is not a failed one, and reporting `false` there would
792/// make an inbound-only link look broken every time.
793///
794/// ```
795/// # use std::collections::BTreeMap;
796/// # use std::sync::{Arc, Mutex, Weak};
797/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
798/// # use majordomus_cli::mesh::cooperation::*;
799/// # use majordomus_cli::mesh::identity::NodeIdentity;
800/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
801/// # use majordomus_cli::mesh::registry::MeshRegistry;
802/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
803/// # use majordomus_cli::mesh::trust::TrustPolicy;
804/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
805/// # impl LinkTransport for Net {
806/// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
807/// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
808/// #         .ok_or_else(|| format!("{e}: nobody there"))?;
809/// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
810/// # let net = Arc::new(Net::default());
811/// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
812/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
813/// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
814/// #     version: "doc".into(), config: CooperationConfig::default(),
815/// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
816/// #     registry: Arc::new(MeshRegistry::new()),
817/// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
818/// #     checkout: CheckoutFacts::default() }).unwrap();
819/// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
820/// # let (a, b) = (runtime("a:1"), runtime("b:1"));
821/// a.dial(&["b:1".into()]).unwrap();
822///
823/// let dialer: &PeerVerdict = &a.verify().peers[0];
824/// assert_eq!(dialer.round_trip, Some(true), "asked, and answered");
825///
826/// let answerer = &b.verify().peers[0];
827/// assert!(!answerer.dialed, "B does not dial A, so B asks nothing of it");
828/// assert_eq!(answerer.round_trip, None, "and reports no verdict it did not earn");
829/// ```
830#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
831pub struct PeerVerdict {
832    /// The peer's runtime key.
833    pub runtime: String,
834    /// The link's standing after the round.
835    pub state: LinkState,
836    /// Whether this runtime dials the peer (and so ran a round with it now).
837    pub dialed: bool,
838    /// Whether the round succeeded; `None` when the peer dials this runtime instead.
839    #[serde(default, skip_serializing_if = "Option::is_none")]
840    pub round_trip: Option<bool>,
841    /// The round trip, milliseconds.
842    #[serde(default, skip_serializing_if = "Option::is_none")]
843    pub rtt_ms: Option<u64>,
844    /// Whether both sides hold the same high-water mark of every stream after the round.
845    #[serde(default, skip_serializing_if = "Option::is_none")]
846    pub converged: Option<bool>,
847    /// What happened.
848    pub detail: String,
849}
850
851/// What `mesh verify` found: the local checks, every peer's round, and the state digest
852/// afterwards. The digest is what makes the report worth running on two machines — two
853/// runtimes that print the same digest have the same sessions, claims, handovers and
854/// reviews, and two that do not have something to reconcile, whatever their links say.
855///
856/// `ok` is the conjunction of everything below it, so a caller may act on the one field
857/// and a person may read the rest to find out which part of it was false.
858///
859/// ```
860/// # use std::sync::Arc;
861/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
862/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
863/// # use majordomus_cli::mesh::identity::NodeIdentity;
864/// # use majordomus_cli::mesh::link::HttpTransport;
865/// # use majordomus_cli::mesh::registry::MeshRegistry;
866/// # use majordomus_cli::mesh::repository::of_root_commits;
867/// # let cooperation = Cooperation::new(CooperationSetup {
868/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
869/// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
870/// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
871/// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
872/// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
873/// # }).unwrap();
874/// use majordomus_cli::mesh::cooperation::MeshVerifyReport;
875///
876/// let report: MeshVerifyReport = cooperation.verify();
877/// assert_eq!(report.runtime.as_deref(), Some(cooperation.runtime_key()));
878/// assert_eq!(report.digest, Some(cooperation.state().digest), "what a peer is compared to");
879/// assert_eq!(report.ok, report.checks.iter().all(|c| c.ok), "one field for every check");
880/// ```
881#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
882pub struct MeshVerifyReport {
883    /// Whether every check holds and every round succeeded.
884    pub ok: bool,
885    /// This runtime.
886    #[serde(default, skip_serializing_if = "Option::is_none")]
887    pub runtime: Option<String>,
888    /// The checks, in the order they ran.
889    pub checks: Vec<MeshVerifyCheck>,
890    /// Every linked peer's verdict.
891    pub peers: Vec<PeerVerdict>,
892    /// The state digest after the rounds.
893    #[serde(default, skip_serializing_if = "Option::is_none")]
894    pub digest: Option<String>,
895}
896
897impl MeshVerifyReport {
898    /// The report of a runtime where cooperation does not run.
899    ///
900    /// Not running is a failing verification rather than an empty one: a person who asked
901    /// whether the mesh works has been told no, and the single check carries the impact —
902    /// nothing leaves or reaches this runtime — and the steps that would turn it on.
903    ///
904    /// ```
905    /// use majordomus_cli::mesh::cooperation::MeshVerifyReport;
906    ///
907    /// let report = MeshVerifyReport::inactive("cooperation is disabled in the declaration");
908    /// assert!(!report.ok, "a mesh that does not run has not been verified");
909    /// let check = &report.checks[0];
910    /// assert!(check.impact.is_some() && check.remediation.is_some(), "what it costs, and the fix");
911    /// ```
912    pub fn inactive(reason: &str) -> Self {
913        MeshVerifyReport {
914            ok: false,
915            runtime: None,
916            checks: vec![MeshVerifyCheck {
917                check: "cooperation".into(),
918                ok: false,
919                detail: reason.into(),
920                impact: Some("no session, claim, handover or review leaves or reaches this runtime".into()),
921                remediation: Some("enable the mesh declaration (enabled: true), trust the peers' keys (trust.allow, from `majordomus mesh identity` on each machine), and restart the server: majordomus serve stop && majordomus serve ensure".into()),
922            }],
923            peers: Vec::new(),
924            digest: None,
925        }
926    }
927}
928
929/// What an operation wrote, as its answer to the caller. Every mesh operation is an event
930/// appended to this runtime's journal, and the caller is handed the three facts it cannot
931/// work out for itself: the mesh-wide key of the thing it created, the event that says so,
932/// and the Lamport stamp that places it against everyone else's events.
933///
934/// The key is what a later operation is addressed by — a claim is released by the key its
935/// acquisition returned, never by the local name it was given — and the stamp is what a
936/// caller waiting for a peer to catch up compares against.
937///
938/// ```
939/// # use std::sync::Arc;
940/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
941/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
942/// # use majordomus_cli::mesh::identity::NodeIdentity;
943/// # use majordomus_cli::mesh::link::HttpTransport;
944/// # use majordomus_cli::mesh::registry::MeshRegistry;
945/// # use majordomus_cli::mesh::repository::of_root_commits;
946/// # let cooperation = Cooperation::new(CooperationSetup {
947/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
948/// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
949/// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
950/// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
951/// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
952/// # }).unwrap();
953/// use majordomus_cli::mesh::cooperation::Written;
954/// use majordomus_cli::mesh::journal::{ClaimMode, Durability, SessionInfo};
955///
956/// let session = SessionInfo::named("s1", "cli");
957/// let written: Written = cooperation
958///     .claim(&session, vec!["docs".into()], None, ClaimMode::Advisory, None)
959///     .unwrap();
960/// assert!(written.lamport > 0, "placed against every other runtime's events");
961/// assert_eq!(written.durability, Durability::Memory, "this journal has no file");
962///
963/// // and the key it returned is how the claim is addressed from here on
964/// cooperation.release(&written.key).unwrap();
965/// ```
966#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
967pub struct Written {
968    /// `<stream>/<local id>` of what was created or changed.
969    pub key: String,
970    /// `<stream>/<seq>` of the event that says so.
971    pub event: String,
972    /// The event's Lamport stamp.
973    pub lamport: u64,
974    /// Whether the event reached this machine's disk: `synced`, `memory` (a journal with no
975    /// file, by configuration), or `not_durable` — held and replicated to peers, not on disk
976    /// here, and not acknowledged as durable. `unknown` from an executable that predates it.
977    #[serde(default)]
978    pub durability: Durability,
979}
980
981/// What a dial or a sync round did not do. The three are kept apart because they are acted
982/// on differently: an unreachable peer is retried with a backoff, a refusal is a decision
983/// that will be made the same way next time and is recorded rather than retried, and the
984/// absence of a link is this runtime's own state and not the peer's.
985///
986/// ```
987/// # use std::sync::Arc;
988/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
989/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
990/// # use majordomus_cli::mesh::identity::NodeIdentity;
991/// # use majordomus_cli::mesh::link::HttpTransport;
992/// # use majordomus_cli::mesh::registry::MeshRegistry;
993/// # use majordomus_cli::mesh::repository::of_root_commits;
994/// # let cooperation = Cooperation::new(CooperationSetup {
995/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
996/// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
997/// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
998/// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
999/// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1000/// # }).unwrap();
1001/// use majordomus_cli::mesh::cooperation::RoundError;
1002///
1003/// // nobody answers at the discard port: a peer to back off from, not a decision
1004/// let nobody = cooperation.dial(&["127.0.0.1:9".into()]).unwrap_err();
1005/// assert!(matches!(nobody, RoundError::Unreachable(_)), "{nobody}");
1006/// assert!(nobody.to_string().contains("127.0.0.1:9"), "a failure names where it was going");
1007///
1008/// // and syncing with a peer this runtime never linked to is its own state, not the peer's
1009/// let never = cooperation.sync_with("somebody-0000000000000002").unwrap_err();
1010/// assert!(matches!(never, RoundError::NoLink), "{never}");
1011/// ```
1012#[derive(Debug, Clone)]
1013pub enum RoundError {
1014    /// No endpoint answered.
1015    Unreachable(String),
1016    /// The peer, or this runtime's check of the peer, refused.
1017    Refused(LinkRefusal),
1018    /// There is no outbound link to sync.
1019    NoLink,
1020}
1021
1022impl std::fmt::Display for RoundError {
1023    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1024        match self {
1025            RoundError::Unreachable(e) => write!(f, "unreachable: {e}"),
1026            RoundError::Refused(r) => write!(f, "refused: {r}"),
1027            RoundError::NoLink => f.write_str("no link"),
1028        }
1029    }
1030}
1031
1032struct Target {
1033    id: String,
1034    key: Option<String>,
1035    endpoints: Vec<String>,
1036}
1037
1038/// A registry record this runtime may dial, under the runtime key it would be dialed by.
1039///
1040/// Its order is the order [`MAX_TARGETS_PER_NODE`] cuts a node's runtimes in: by node, then
1041/// the runtimes answering now before the ones that stopped, then by runtime. Presence comes
1042/// before the runtime because the registry keeps a stopped runtime's record for its retention
1043/// and every runtime of a machine orders the records alike: cut by runtime alone, stopped
1044/// servers that sort first fill the cap on every live runtime at once, and no live runtime of
1045/// the node dials a live sibling until the stopped records expire.
1046struct DialCandidate<'a> {
1047    record: &'a NodeRecord,
1048    key: String,
1049}
1050
1051impl crate::order::Ordered for DialCandidate<'_> {
1052    fn order_key(&self) -> crate::order::OrderKey<'_> {
1053        let rank = match self.record.presence {
1054            Presence::Present => 0,
1055            Presence::Absent => 1,
1056        };
1057        crate::order::OrderKey::grouped(
1058            self.record.node_id.as_str(),
1059            &self.record.runtime,
1060            &self.key,
1061        )
1062        .ranked(rank)
1063    }
1064}
1065
1066/// The registry records this runtime dials: every record of `repository` that names a
1067/// runtime slot and an endpoint, whose key `trusted` accepts, other than `own_key` — at most
1068/// [`MAX_TARGETS_PER_NODE`] of one node, taken in [`DialCandidate`] order, so a node's present
1069/// runtimes are dialed before its stopped ones. The same records in any order choose the
1070/// same targets.
1071fn dial_targets(
1072    records: &[NodeRecord],
1073    repository: &str,
1074    own_key: &str,
1075    trusted: impl Fn(&str) -> bool,
1076) -> Vec<Target> {
1077    let mut candidates: Vec<DialCandidate<'_>> = records
1078        .iter()
1079        .filter(|record| {
1080            !record.runtime.is_empty()
1081                && !record.endpoints.is_empty()
1082                && record.repositories.iter().any(|r| r == repository)
1083        })
1084        .map(|record| DialCandidate {
1085            key: format!("{}-{}", record.node_id, record.runtime),
1086            record,
1087        })
1088        .filter(|candidate| candidate.key != own_key && trusted(&candidate.record.public_key))
1089        .collect();
1090    crate::order::canonical(&mut candidates);
1091    // At most MAX_TARGETS_PER_NODE runtimes of one node are dialed: runtime slots are free
1092    // to invent, and a key must not turn this runtime into a dialer of arbitrary endpoints.
1093    let mut per_node: BTreeMap<&str, usize> = BTreeMap::new();
1094    let mut targets = Vec::new();
1095    for candidate in candidates {
1096        let dialed = per_node
1097            .entry(candidate.record.node_id.as_str())
1098            .or_default();
1099        if *dialed >= MAX_TARGETS_PER_NODE {
1100            continue;
1101        }
1102        *dialed += 1;
1103        targets.push(Target {
1104            id: candidate.key.clone(),
1105            key: Some(candidate.key),
1106            endpoints: candidate.record.endpoints.clone(),
1107        });
1108    }
1109    targets
1110}
1111
1112/// One runtime's whole part in the mesh, behind one handle. It holds the single journal
1113/// this process writes to, the single table of links it has, and the admission lock that
1114/// decides claims — and it is deliberately one object rather than a set of services,
1115/// because every one of those is the other's context: a claim is admitted against state
1116/// folded from events that arrived over a link, and a link is admitted against trust the
1117/// same object evaluates.
1118///
1119/// Every surface — the command line, HTTP, MCP — performs its operations through this, so
1120/// there is one place where a session is opened and one where a claim is refused, whoever
1121/// asked. Nothing here dials or beats until [`Cooperation::start`]; a runtime that is
1122/// merely built is a complete, readable, local mesh of one.
1123///
1124/// ```
1125/// # use std::sync::Arc;
1126/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1127/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1128/// # use majordomus_cli::mesh::identity::NodeIdentity;
1129/// # use majordomus_cli::mesh::link::HttpTransport;
1130/// # use majordomus_cli::mesh::registry::MeshRegistry;
1131/// # use majordomus_cli::mesh::repository::of_root_commits;
1132/// # let cooperation: Arc<Cooperation> = Cooperation::new(CooperationSetup {
1133/// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1134/// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1135/// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1136/// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
1137/// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1138/// # }).unwrap();
1139/// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
1140///
1141/// let session = SessionInfo::named("s1", "cli");
1142/// cooperation.claim(&session, vec!["apps".into()], Some("documenting".into()),
1143///     ClaimMode::Exclusive, None).unwrap();
1144///
1145/// // the operation, the event and the folded state are one runtime's, with nobody linked
1146/// let state = cooperation.state();
1147/// assert_eq!(state.claims[0].intent.as_deref(), Some("documenting"));
1148/// assert_eq!(state.sessions.len(), 1, "claiming opened the session it needed");
1149/// ```
1150pub struct Cooperation {
1151    identity: Arc<NodeIdentity>,
1152    card: RuntimeCard,
1153    own_key: String,
1154    repository: MeshRepositoryIdentity,
1155    config: CooperationConfig,
1156    /// The trust in force: replaced, never patched, when the declaration changes under a
1157    /// running server ([`Cooperation::retrust`]).
1158    trust: std::sync::RwLock<TrustConfig>,
1159    registry: Arc<MeshRegistry>,
1160    transport: Arc<dyn LinkTransport>,
1161    journal: Arc<Journal>,
1162    table: Mutex<Table>,
1163    counters: Counters,
1164    stop: Arc<AtomicBool>,
1165    started_at: String,
1166    board: Option<Arc<crate::peers::PeerBoard>>,
1167    checkout: CheckoutFacts,
1168    projected: Mutex<BTreeMap<String, Projected>>,
1169    admission: Mutex<()>,
1170    rounds: Mutex<BTreeMap<String, Arc<Mutex<()>>>>,
1171}
1172
1173/// The most refusals the table lists: a stream of hellos from invented keys must not grow
1174/// memory without bound. The oldest refusal goes first.
1175const MAX_REFUSED: usize = 128;
1176
1177/// The most runtimes of one node this runtime dials.
1178pub const MAX_TARGETS_PER_NODE: usize = 8;
1179
1180/// `text` cut to at most `max` bytes on a character boundary, with control characters
1181/// removed: what the journal's single-line bounds accept, so a projection never fails
1182/// validation on a long or multi-byte title and silently retries forever.
1183fn clip(text: &str, max: usize) -> String {
1184    let mut out = String::new();
1185    for c in text.chars().filter(|c| !c.is_control()) {
1186        if out.len() + c.len_utf8() > max {
1187            break;
1188        }
1189        out.push(c);
1190    }
1191    out
1192}
1193
1194fn remember_refusal(table: &mut Table, key: String, refused: Refused) {
1195    if !table.refused.contains_key(&key) && table.refused.len() >= MAX_REFUSED {
1196        if let Some(oldest) = table
1197            .refused
1198            .iter()
1199            .min_by(|a, b| a.1.at.cmp(&b.1.at))
1200            .map(|(k, _)| k.clone())
1201        {
1202            table.refused.remove(&oldest);
1203        }
1204    }
1205    table.refused.insert(key, refused);
1206}
1207
1208fn rfc3339_now() -> String {
1209    crate::peers::rfc3339(std::time::SystemTime::now())
1210}
1211
1212/// The trust verdict for `public_key` on a machine whose own key is `own_key`: the one
1213/// definition behind [`Cooperation::trust_of_key`], and behind the journal's reload, which
1214/// runs before the runtime exists and must decide exactly what the live path decides.
1215fn key_trust(
1216    own_key: &str,
1217    config: &TrustConfig,
1218    registry: &MeshRegistry,
1219    public_key: &str,
1220) -> TrustState {
1221    if public_key == own_key {
1222        return TrustState::Trusted("this machine's own key".into());
1223    }
1224    let Some(node) = node_id_of_key(public_key) else {
1225        return TrustState::Rejected("not a key".into());
1226    };
1227    trust::evaluate(
1228        &config.policy,
1229        &config.allow,
1230        public_key,
1231        registry.known_key(&node).as_deref(),
1232        registry.trust_of(&node).as_ref(),
1233    )
1234}
1235
1236impl Cooperation {
1237    /// Open the journal and build the runtime; nothing dials until [`Cooperation::start`].
1238    ///
1239    /// The declaration is validated and this runtime's card is checked here rather than at
1240    /// the first handshake, so a runtime that would be refused by every peer refuses to
1241    /// exist instead of failing one link at a time in the background where nobody reads
1242    /// it. Building it is also what fixes this runtime's stream: the key it answers to
1243    /// from now on, and the stream its events are written to.
1244    ///
1245    /// ```
1246    /// # use std::sync::Arc;
1247    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1248    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1249    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1250    /// # use majordomus_cli::mesh::link::HttpTransport;
1251    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1252    /// # use majordomus_cli::mesh::repository::of_root_commits;
1253    /// # let setup = |heartbeat: u64, expiry: u64| CooperationSetup {
1254    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1255    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1256    /// #     version: "doc".into(),
1257    /// #     config: CooperationConfig { heartbeat_seconds: heartbeat, expiry_seconds: expiry,
1258    /// #         ..CooperationConfig::default() },
1259    /// #     trust: TrustConfig::default(), journal_path: None,
1260    /// #     registry: Arc::new(MeshRegistry::new()), transport: Arc::new(HttpTransport),
1261    /// #     board: None, checkout: CheckoutFacts::default() };
1262    /// let cooperation = Cooperation::new(setup(5, 30)).unwrap();
1263    /// assert!(cooperation.peers().is_empty(), "built, and dialing nobody");
1264    /// assert!(cooperation.journal().own_stream().runtime_key() == cooperation.runtime_key());
1265    ///
1266    /// // a declaration that no peer could live with is refused here, not at the first link
1267    /// let Err(refused) = Cooperation::new(setup(30, 5)) else { panic!("a live runtime") };
1268    /// assert!(refused.to_string().contains("expiry"), "{refused}");
1269    /// ```
1270    pub fn new(setup: CooperationSetup) -> Result<Arc<Self>, super::MeshError> {
1271        setup.config.validate().map_err(super::MeshError::Config)?;
1272        // The journal file is believed as far as the declared trust believes its origins
1273        // now, exactly as `origin_accept` decides for an event arriving over a link: a key
1274        // withdrawn from the allowlist does not come back from this runtime's own disk.
1275        let own_pk = setup.identity.public.public_key.as_str();
1276        let reload_accept = |event: &MeshEvent| {
1277            if key_trust(own_pk, &setup.trust, &setup.registry, &event.pk).is_trusted() {
1278                Ok(())
1279            } else {
1280                Err(Rejection::Untrusted)
1281            }
1282        };
1283        let journal = Arc::new(Journal::open_trusting(
1284            Arc::clone(&setup.identity),
1285            &setup.runtime,
1286            setup.repository.id.clone(),
1287            setup.journal_path.clone(),
1288            &reload_accept,
1289        )?);
1290        let reloaded = journal.reload_report();
1291        if reloaded.rejected_total() > 0 {
1292            tracing::warn!(
1293                accepted = reloaded.accepted,
1294                refused = ?reloaded.rejected,
1295                "mesh journal reload refused events its trust no longer admits"
1296            );
1297        }
1298        let card = RuntimeCard {
1299            pk: setup.identity.public.public_key.clone(),
1300            runtime: setup.runtime.clone(),
1301            instance: setup.identity.public.instance_id.as_str().into(),
1302            repo: setup.repository.id.clone(),
1303            name: setup.identity.public.display_name.clone(),
1304            version: setup.version.chars().take(32).collect(),
1305            features: FEATURES.iter().map(|f| f.to_string()).collect(),
1306            endpoints: setup
1307                .endpoints
1308                .iter()
1309                .take(super::protocol::MAX_ENDPOINTS)
1310                .cloned()
1311                .collect(),
1312        };
1313        card.check().map_err(super::MeshError::Protocol)?;
1314        let own_key = journal.own_stream().runtime_key();
1315        Ok(Arc::new(Cooperation {
1316            identity: setup.identity,
1317            card,
1318            own_key,
1319            repository: setup.repository,
1320            config: setup.config,
1321            trust: std::sync::RwLock::new(setup.trust),
1322            registry: setup.registry,
1323            transport: setup.transport,
1324            journal,
1325            table: Mutex::new(Table::default()),
1326            counters: Counters::default(),
1327            stop: Arc::new(AtomicBool::new(false)),
1328            started_at: rfc3339_now(),
1329            board: setup.board,
1330            checkout: setup.checkout,
1331            projected: Mutex::new(BTreeMap::new()),
1332            admission: Mutex::new(()),
1333            rounds: Mutex::new(BTreeMap::new()),
1334        }))
1335    }
1336
1337    /// Project the peer board into the journal: an attached session that is new opens a
1338    /// mesh session, a changed announcement releases its previous advisory claim and
1339    /// takes the new one, a withdrawn announcement is released, and a session that left
1340    /// the board is closed. One-way and idempotent — the board stays the local truth, the
1341    /// journal carries it to every linked runtime. Nothing happens when nothing changed.
1342    ///
1343    /// This is the whole bridge between the board a person reads on one machine and the
1344    /// mesh every other machine reads: a worker announces once, to its own server, and is
1345    /// visible everywhere without knowing that anything else exists.
1346    ///
1347    /// ```
1348    /// # use std::sync::Arc;
1349    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1350    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1351    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1352    /// # use majordomus_cli::mesh::link::HttpTransport;
1353    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1354    /// # use majordomus_cli::mesh::repository::of_root_commits;
1355    /// use majordomus_cli::peers::{ClientInfo, PeerBoard, Transport};
1356    ///
1357    /// let board = Arc::new(PeerBoard::new());
1358    /// # let cooperation = Cooperation::new(CooperationSetup {
1359    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1360    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1361    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1362    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
1363    /// #     transport: Arc::new(HttpTransport), board: Some(Arc::clone(&board)),
1364    /// #     checkout: CheckoutFacts::default() }).unwrap();
1365    /// let worker = board.attach(Transport::Http);
1366    /// board.identify(&worker, ClientInfo { name: "claude-code".into(), version: "1".into(),
1367    ///     title: None });
1368    /// board.announce(&worker, "documenting the mesh", vec!["apps/majordomus-cli".into()]);
1369    ///
1370    /// cooperation.project_board();
1371    /// let state = cooperation.state();
1372    /// assert_eq!(state.claims[0].scope, ["apps/majordomus-cli"], "the announcement crossed");
1373    /// assert_eq!(state.sessions[0].info.client, "claude-code");
1374    ///
1375    /// // idempotent: projecting an unchanged board writes nothing
1376    /// cooperation.project_board();
1377    /// assert_eq!(cooperation.state().digest, state.digest);
1378    ///
1379    /// // and a worker that leaves takes its claim with it
1380    /// board.detach(&worker);
1381    /// cooperation.project_board();
1382    /// assert!(!cooperation.state().claims[0].state.is_live());
1383    /// ```
1384    pub fn project_board(&self) {
1385        let Some(board) = &self.board else { return };
1386        let peers = board.list();
1387        let mut projected = self.projected.lock().expect("cooperation projection");
1388        let mut seen = Vec::new();
1389        for peer in peers.iter().filter(|p| p.attached) {
1390            let session = format!("board-{}", peer.id);
1391            seen.push(session.clone());
1392            // clipped before it is compared: the projection holds what the journal accepted,
1393            // so comparing the board's whole text with it would differ on every pass for an
1394            // intent past the bound, and reopen the session each time
1395            let intent = peer.claims.first().map(|c| clip(&c.intent, 512));
1396            let entry = projected.entry(session.clone()).or_default();
1397            let fresh = entry.info.session.is_empty();
1398            if fresh || entry.info.intent != intent {
1399                let info = SessionInfo {
1400                    session: session.clone(),
1401                    client: clip(&peer.client.name, 64),
1402                    worker: peer
1403                        .client
1404                        .title
1405                        .clone()
1406                        .filter(|t| !t.is_empty())
1407                        .map(|t| clip(&t, 128)),
1408                    intent: intent.clone(),
1409                    checkout: self.checkout.id.clone(),
1410                    branch: self.checkout.git(&["symbolic-ref", "--short", "HEAD"]),
1411                    head: self.checkout.git(&["rev-parse", "--verify", "HEAD"]),
1412                    ..SessionInfo::default()
1413                };
1414                if self.open_session(info.clone()).is_ok() {
1415                    entry.info = info;
1416                }
1417            }
1418            let mut wanted: BTreeMap<String, (Vec<String>, String)> = BTreeMap::new();
1419            for claim in &peer.claims {
1420                let scope: Vec<String> = claim
1421                    .scope
1422                    .iter()
1423                    .map(|p| p.trim().trim_start_matches("./").to_string())
1424                    .filter(|p| !p.is_empty())
1425                    .collect();
1426                if !scope.is_empty() {
1427                    wanted.insert(
1428                        claim.name.clone().unwrap_or_default(),
1429                        (scope, claim.intent.clone()),
1430                    );
1431                }
1432            }
1433            let stale: Vec<String> = entry
1434                .claims
1435                .iter()
1436                .filter(|(name, (_, scope, intent))| {
1437                    wanted
1438                        .get(*name)
1439                        .is_none_or(|(s, i)| s != scope || i != intent)
1440                })
1441                .map(|(name, _)| name.clone())
1442                .collect();
1443            for name in stale {
1444                if let Some((key, _, _)) = entry.claims.remove(&name) {
1445                    let _ = self.release(&key);
1446                }
1447            }
1448            for (name, (scope, claim_intent)) in wanted {
1449                if entry.claims.contains_key(&name) {
1450                    continue;
1451                }
1452                match self.claim(
1453                    &entry.info,
1454                    scope.clone(),
1455                    Some(clip(&claim_intent, 512)).filter(|i| !i.is_empty()),
1456                    ClaimMode::Advisory,
1457                    None,
1458                ) {
1459                    Ok(written) => {
1460                        entry
1461                            .claims
1462                            .insert(name, (written.key, scope, claim_intent));
1463                    }
1464                    Err(e) => {
1465                        tracing::debug!(runtime_id = %self.own_key, session_id = %session, error = %e, "a board announcement was not projected")
1466                    }
1467                }
1468            }
1469        }
1470        let gone: Vec<String> = projected
1471            .keys()
1472            .filter(|s| !seen.contains(s))
1473            .cloned()
1474            .collect();
1475        for session in gone {
1476            if let Some(entry) = projected.remove(&session) {
1477                for (key, _, _) in entry.claims.values() {
1478                    let _ = self.release(key);
1479                }
1480            }
1481            let _ = self.close_session(&session);
1482        }
1483    }
1484
1485    /// This runtime's key, `<node>-<runtime>`: who it is to every other runtime, for as
1486    /// long as this checkout exists on this machine. It survives a restart — which is what
1487    /// lets a peer recognise a returning runtime as the same one rather than a second —
1488    /// while the instance inside its card does not.
1489    pub fn runtime_key(&self) -> &str {
1490        &self.own_key
1491    }
1492
1493    /// The journal this runtime writes to and every peer's events land in. Handed out
1494    /// rather than wrapped because reading the mesh is reading events: a caller that wants
1495    /// marks, tallies or the raw stream takes them from here, and the operations above
1496    /// exist for the writes, which must not be done by hand.
1497    pub fn journal(&self) -> &Arc<Journal> {
1498        &self.journal
1499    }
1500
1501    fn heartbeat(&self) -> Duration {
1502        Duration::from_secs(self.config.heartbeat_seconds)
1503    }
1504
1505    fn expiry(&self) -> Duration {
1506        Duration::from_secs(self.config.expiry_seconds)
1507    }
1508
1509    /// Start the supervisor: the heartbeat, the dialers and the expiry. Never blocks.
1510    ///
1511    /// It returns before anything has linked, because linking depends on other machines
1512    /// and a server must answer while it is still finding them. A disabled declaration
1513    /// starts nothing at all and says so through [`Cooperation::status`], rather than
1514    /// starting a supervisor that would refuse every round.
1515    ///
1516    /// ```
1517    /// # use std::sync::Arc;
1518    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1519    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1520    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1521    /// # use majordomus_cli::mesh::link::HttpTransport;
1522    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1523    /// # use majordomus_cli::mesh::repository::of_root_commits;
1524    /// # let cooperation = Cooperation::new(CooperationSetup {
1525    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1526    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1527    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1528    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
1529    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1530    /// # }).unwrap();
1531    /// cooperation.start();
1532    /// assert!(!cooperation.stopped(), "the supervisor is up and the caller was not held");
1533    ///
1534    /// cooperation.stop();
1535    /// assert!(cooperation.state().digest.len() == 32, "and the journal is still readable");
1536    /// ```
1537    pub fn start(self: &Arc<Self>) {
1538        if !self.config.enabled {
1539            return;
1540        }
1541        let me = Arc::clone(self);
1542        me.counters.threads.fetch_add(1, Ordering::SeqCst);
1543        let _ = std::thread::Builder::new()
1544            .name("majordomus-mesh-cooperation".into())
1545            .spawn(move || {
1546                tracing::info!(runtime_id = %me.own_key, repository_id = %me.repository.id, "mesh cooperation started");
1547                let mut tick: u64 = 0;
1548                while !me.stop.load(Ordering::SeqCst) {
1549                    me.journal.beat_own();
1550                    me.project_board();
1551                    me.reconcile_workers();
1552                    me.expire_links();
1553                    if tick % 60 == 59 {
1554                        me.journal
1555                            .compact(me.expiry(), PEER_RETENTION, HANDOVER_RETENTION, false);
1556                    }
1557                    tick += 1;
1558                    me.sleep(me.heartbeat());
1559                }
1560                me.counters.threads.fetch_sub(1, Ordering::SeqCst);
1561            });
1562    }
1563
1564    /// Stop every cooperation thread at its next bounded wait and drop every link. The
1565    /// journal stays readable; this runtime's stream simply stops beating.
1566    ///
1567    /// Stopping is deliberately not a farewell. Nothing is said to the peers, because a
1568    /// runtime that crashes says nothing either and the mesh must behave the same way in
1569    /// both cases: the streams stop beating, the claims stop excluding after the expiry,
1570    /// and no peer is left believing a promise that a power cut would have broken.
1571    ///
1572    /// ```
1573    /// # use std::sync::Arc;
1574    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1575    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1576    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1577    /// # use majordomus_cli::mesh::link::HttpTransport;
1578    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1579    /// # use majordomus_cli::mesh::repository::of_root_commits;
1580    /// # let cooperation = Cooperation::new(CooperationSetup {
1581    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1582    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1583    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1584    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
1585    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1586    /// # }).unwrap();
1587    /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
1588    ///
1589    /// let session = SessionInfo::named("s1", "cli");
1590    /// cooperation.claim(&session, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
1591    /// cooperation.stop();
1592    ///
1593    /// assert!(cooperation.stopped());
1594    /// assert_eq!(cooperation.state().claims.len(), 1, "the record survives the runtime");
1595    /// ```
1596    ///
1597    /// [`begin_stop`](Self::begin_stop) is the half of this that takes no lock, for a caller
1598    /// that must not be delayed.
1599    pub fn stop(&self) {
1600        self.begin_stop();
1601        let mut table = self.table.lock().expect("cooperation table");
1602        for flag in table.workers.values() {
1603            flag.store(true, Ordering::SeqCst);
1604        }
1605        table.workers.clear();
1606        table.inbound_links.clear();
1607        for peer in table.peers.values_mut() {
1608            peer.outbound = None;
1609            peer.inbound = None;
1610        }
1611    }
1612
1613    /// Tell every part of this runtime to stop, and return at once.
1614    ///
1615    /// [`stop`](Self::stop) also empties the link table, and to do that it must take the
1616    /// table's lock — which a worker can be holding across a dial that waits out the link
1617    /// timeout. A server shutting down must not wait for that: it has a lease to release,
1618    /// and a lease released late reads to the next process as a server that is still there.
1619    /// So the flag, which every loop and every handler checks, is set without any lock, and
1620    /// the draining is left to `stop`.
1621    ///
1622    /// ```
1623    /// # use std::sync::Arc;
1624    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1625    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1626    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1627    /// # use majordomus_cli::mesh::link::HttpTransport;
1628    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1629    /// # use majordomus_cli::mesh::repository::of_root_commits;
1630    /// # let cooperation = Cooperation::new(CooperationSetup {
1631    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1632    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1633    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1634    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
1635    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1636    /// # }).unwrap();
1637    /// cooperation.begin_stop();
1638    /// assert!(cooperation.stopped(), "every loop and handler sees it immediately");
1639    /// ```
1640    pub fn begin_stop(&self) {
1641        self.stop.store(true, Ordering::SeqCst);
1642    }
1643
1644    /// Whether this runtime has been told to stop. It is checked at the top of every
1645    /// handler, so a stopped runtime refuses a hello and a sync with a typed refusal
1646    /// instead of quietly half-linking while its threads wind down.
1647    ///
1648    /// ```
1649    /// # use std::sync::Arc;
1650    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1651    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1652    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1653    /// # use majordomus_cli::mesh::link::HttpTransport;
1654    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1655    /// # use majordomus_cli::mesh::repository::of_root_commits;
1656    /// # let cooperation = Cooperation::new(CooperationSetup {
1657    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1658    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1659    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1660    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
1661    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1662    /// # }).unwrap();
1663    /// use majordomus_cli::mesh::link::{RefusalCode, Signed};
1664    ///
1665    /// assert!(!cooperation.stopped());
1666    /// cooperation.stop();
1667    ///
1668    /// let hello = Signed { body: serde_json::json!({}), sig: String::new() };
1669    /// let reply = cooperation.accept_hello(&hello);
1670    /// assert_eq!(reply.refusal.unwrap().code, RefusalCode::NotActive, "refused, not ignored");
1671    /// ```
1672    pub fn stopped(&self) -> bool {
1673        self.stop.load(Ordering::SeqCst)
1674    }
1675
1676    fn sleep(&self, total: Duration) {
1677        let mut slept = Duration::ZERO;
1678        while slept < total && !self.stop.load(Ordering::SeqCst) {
1679            let slice = SLICE.min(total - slept);
1680            std::thread::sleep(slice);
1681            slept += slice;
1682        }
1683    }
1684
1685    // ------------------------------------------------------------------ trust
1686
1687    /// The trust verdict for a key: this machine's own key is trusted as itself; any
1688    /// other by the declared policy against what discovery knows of the node.
1689    ///
1690    /// It is asked twice for every event, once when a link is admitted and again when an
1691    /// event is ingested, because a link admitted an hour ago is not a warrant for what
1692    /// arrives over it now — an event carries the key that signed it, and a key that has
1693    /// since been withdrawn from the allowlist stops being believed without dropping
1694    /// anything.
1695    ///
1696    /// ```
1697    /// # use std::sync::Arc;
1698    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1699    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1700    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1701    /// # use majordomus_cli::mesh::link::HttpTransport;
1702    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1703    /// # use majordomus_cli::mesh::repository::of_root_commits;
1704    /// # let identity = Arc::new(NodeIdentity::ephemeral().unwrap());
1705    /// # let own_key = identity.public.public_key.clone();
1706    /// # let cooperation = Cooperation::new(CooperationSetup {
1707    /// #     identity, runtime: "0000000000000001".into(),
1708    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1709    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1710    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
1711    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1712    /// # }).unwrap();
1713    /// let stranger = NodeIdentity::ephemeral().unwrap().public.public_key.clone();
1714    ///
1715    /// // two runtimes of one machine share a key, and neither has to allow the other
1716    /// assert!(cooperation.trust_of_key(&own_key).is_trusted(), "this machine, twice over");
1717    ///
1718    /// // and under the default policy an unheard-of key is not trusted by being reachable
1719    /// assert!(!cooperation.trust_of_key(&stranger).is_trusted());
1720    /// assert!(!cooperation.trust_of_key("not a key at all").is_trusted());
1721    /// ```
1722    pub fn trust_of_key(&self, public_key: &str) -> TrustState {
1723        key_trust(
1724            &self.identity.public.public_key,
1725            &self
1726                .trust
1727                .read()
1728                .unwrap_or_else(std::sync::PoisonError::into_inner),
1729            &self.registry,
1730            public_key,
1731        )
1732    }
1733
1734    /// Put `trust` in force while this runtime runs, and end every link to a runtime it no
1735    /// longer trusts. Returns the runtime keys whose links ended.
1736    ///
1737    /// Before this, a key withdrawn from the declaration kept its links until every runtime
1738    /// that had admitted it restarted. Now the server calls this whenever it reads a new
1739    /// generation of the repository: the withdrawn peer's link is gone at once, its next
1740    /// sync meets `unknown_link`, and the hello it says next is refused `untrusted`. Events it
1741    /// wrote stay in the journal until the next restart, whose reload applies the trust in
1742    /// force (I2124).
1743    ///
1744    /// ```
1745    /// use majordomus_cli::mesh::config::TrustConfig;
1746    /// # use std::sync::Arc;
1747    /// # use majordomus_cli::mesh::cooperation::{Cooperation, CooperationSetup, CheckoutFacts};
1748    /// # use majordomus_cli::mesh::{identity::NodeIdentity, registry::MeshRegistry, repository::of_root_commits};
1749    /// # use majordomus_cli::mesh::config::CooperationConfig;
1750    /// # use majordomus_cli::mesh::trust::TrustPolicy;
1751    /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed};
1752    /// # struct Nowhere;
1753    /// # impl LinkTransport for Nowhere {
1754    /// #     fn post(&self, e: &str, _: &str, _: &Signed) -> Result<LinkReply, String> { Err(format!("{e}: nobody")) } }
1755    /// # let c = Cooperation::new(CooperationSetup {
1756    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1757    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec![], version: "doc".into(),
1758    /// #     config: CooperationConfig::default(),
1759    /// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
1760    /// #     registry: Arc::new(MeshRegistry::new()), transport: Arc::new(Nowhere), board: None,
1761    /// #     checkout: CheckoutFacts::default() }).unwrap();
1762    /// let stranger = NodeIdentity::ephemeral().unwrap().public.public_key;
1763    /// assert!(c.trust_of_key(&stranger).is_trusted(), "tofu");
1764    /// assert!(c.retrust(TrustConfig::default()).is_empty(), "no link to end");
1765    /// assert!(!c.trust_of_key(&stranger).is_trusted(), "deny_unknown is in force now");
1766    /// ```
1767    pub fn retrust(&self, trust: TrustConfig) -> Vec<String> {
1768        *self
1769            .trust
1770            .write()
1771            .unwrap_or_else(std::sync::PoisonError::into_inner) = trust;
1772        let mut table = self.table.lock().expect("cooperation table");
1773        let mut ended = Vec::new();
1774        let mut links = Vec::new();
1775        for (key, peer) in table.peers.iter_mut() {
1776            let verdict = self.trust_of_key(&peer.card.pk);
1777            if verdict.is_trusted() {
1778                continue;
1779            }
1780            if let Some(inbound) = peer.inbound.take() {
1781                links.push(inbound.link);
1782            }
1783            peer.outbound = None;
1784            peer.expired = true;
1785            peer.trust = verdict;
1786            ended.push(key.clone());
1787            tracing::warn!(runtime_id = %self.own_key, peer = %key, "mesh link ended: the peer's key is no longer trusted");
1788        }
1789        for link in links {
1790            table.inbound_links.remove(&link);
1791        }
1792        ended
1793    }
1794
1795    fn origin_accept(&self, event: &MeshEvent) -> Result<(), Rejection> {
1796        if self.trust_of_key(&event.pk).is_trusted() {
1797            Ok(())
1798        } else {
1799            Err(Rejection::Untrusted)
1800        }
1801    }
1802
1803    fn count_refusal(map: &Mutex<BTreeMap<RefusalCode, u64>>, code: RefusalCode) {
1804        *map.lock()
1805            .expect("refusal counters")
1806            .entry(code)
1807            .or_default() += 1;
1808    }
1809
1810    fn refuse_in(&self, code: RefusalCode, detail: String, runtime: Option<String>) -> LinkReply {
1811        Self::count_refusal(&self.counters.refused_in, code);
1812        tracing::warn!(runtime_id = %self.own_key, peer = runtime.as_deref().unwrap_or("?"), code = code.as_str(), detail = %detail, "mesh link refused");
1813        let reply = LinkReply::refused(code, detail);
1814        if let (Some(refusal), Some(runtime)) = (reply.refusal.clone(), runtime) {
1815            let mut table = self.table.lock().expect("cooperation table");
1816            remember_refusal(
1817                &mut table,
1818                format!("inbound:{runtime}"),
1819                Refused {
1820                    refusal,
1821                    runtime: Some(runtime),
1822                    direction: "inbound",
1823                    at: rfc3339_now(),
1824                },
1825            );
1826        }
1827        reply
1828    }
1829
1830    // ------------------------------------------------------------------ answering
1831
1832    /// Answer a hello: the whole admission, in order — shape, signature, freshness,
1833    /// replay, protocol, self, repository, trust, capacity — and a signed welcome.
1834    ///
1835    /// The order is the point. Everything cheap and unauthenticated is decided before
1836    /// anything expensive, and nothing a stranger says is believed before its signature is
1837    /// checked, so an unsigned flood costs a parse and a refusal. It answers a refusal
1838    /// rather than dropping the message, because a peer that is being refused for its
1839    /// clock, its repository or its key can only fix that if it is told.
1840    ///
1841    /// ```
1842    /// # use std::collections::BTreeMap;
1843    /// # use std::sync::{Arc, Mutex, Weak};
1844    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1845    /// # use majordomus_cli::mesh::cooperation::*;
1846    /// # use majordomus_cli::mesh::identity::NodeIdentity;
1847    /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, RefusalCode, Signed, HELLO_PATH};
1848    /// # use majordomus_cli::mesh::registry::MeshRegistry;
1849    /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
1850    /// # use majordomus_cli::mesh::trust::TrustPolicy;
1851    /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
1852    /// # impl LinkTransport for Net {
1853    /// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
1854    /// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
1855    /// #         .ok_or_else(|| format!("{e}: nobody there"))?;
1856    /// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
1857    /// # let net = Arc::new(Net::default());
1858    /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
1859    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
1860    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
1861    /// #     version: "doc".into(), config: CooperationConfig::default(),
1862    /// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
1863    /// #     registry: Arc::new(MeshRegistry::new()),
1864    /// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
1865    /// #     checkout: CheckoutFacts::default() }).unwrap();
1866    /// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
1867    /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
1868    /// // A dials; the transport hands B's accept_hello the signed hello, and B welcomes it
1869    /// a.dial(&["b:1".into()]).unwrap();
1870    /// assert_eq!(b.peers()[0].runtime, a.runtime_key(), "admitted, both ways");
1871    ///
1872    /// // anything that is not a signed hello of this protocol is refused, and named
1873    /// let nonsense = Signed { body: serde_json::json!({"hello": true}), sig: "00".into() };
1874    /// let refusal = b.accept_hello(&nonsense).refusal.unwrap();
1875    /// assert_eq!(refusal.code, RefusalCode::Malformed);
1876    /// ```
1877    pub fn accept_hello(&self, message: &Signed) -> LinkReply {
1878        if self.stopped() {
1879            return LinkReply::refused(RefusalCode::NotActive, "cooperation is stopped");
1880        }
1881        if serde_json::to_vec(message)
1882            .map(|b| b.len())
1883            .unwrap_or(usize::MAX)
1884            > MAX_LINK_MESSAGE
1885        {
1886            return self.refuse_in(
1887                RefusalCode::Oversized,
1888                "the hello exceeds the message bound".into(),
1889                None,
1890            );
1891        }
1892        let hello: Hello = match serde_json::from_value(message.body.clone()) {
1893            Ok(h) => h,
1894            Err(e) => {
1895                return self.refuse_in(RefusalCode::Malformed, format!("not a hello: {e}"), None)
1896            }
1897        };
1898        if let Err(e) = hello.card.check() {
1899            return self.refuse_in(RefusalCode::Malformed, e, None);
1900        }
1901        let runtime = hello.card.runtime_key();
1902        if !verify_signed(&hello.card.pk, Domain::Hello, message) {
1903            return self.refuse_in(
1904                RefusalCode::Signature,
1905                "the hello's signature does not verify".into(),
1906                runtime,
1907            );
1908        }
1909        if hello.ts.abs_diff(super::protocol::now()) > MAX_LINK_SKEW_SECONDS {
1910            return self.refuse_in(
1911                RefusalCode::Stale,
1912                "the hello's clock is outside the skew window".into(),
1913                runtime,
1914            );
1915        }
1916        if hello.nonce.len() != 32 {
1917            return self.refuse_in(
1918                RefusalCode::Malformed,
1919                "the nonce is not 32 hex characters".into(),
1920                runtime,
1921            );
1922        }
1923        let Some(proto) = negotiate(hello.proto_min, hello.proto_max) else {
1924            return self.refuse_in(
1925                RefusalCode::ProtocolUnsupported,
1926                format!(
1927                    "this runtime speaks link protocol {LINK_PROTOCOL_MIN}..{LINK_PROTOCOL_MAX}; the peer speaks {}..{}",
1928                    hello.proto_min, hello.proto_max
1929                ),
1930                runtime,
1931            );
1932        };
1933        let runtime_key = runtime.clone().unwrap_or_default();
1934        if runtime_key == self.own_key {
1935            return self.refuse_in(
1936                RefusalCode::SelfLink,
1937                "that is this runtime".into(),
1938                runtime,
1939            );
1940        }
1941        // An addressed hello is this runtime's only when it names this runtime. The address is
1942        // inside the signed bytes, so a copy taken on its way to another runtime of the same
1943        // repository cannot be replayed here; refused before the replay cache sees its nonce.
1944        if let Some(to) = hello.to.as_deref() {
1945            if to != self.own_key {
1946                return self.refuse_in(
1947                    RefusalCode::Misaddressed,
1948                    "the hello is addressed to another runtime".into(),
1949                    runtime,
1950                );
1951            }
1952        }
1953        if hello.card.repo != self.repository.id {
1954            return self.refuse_in(
1955                RefusalCode::RepositoryMismatch,
1956                format!(
1957                    "this runtime serves repository {}; the peer serves {}",
1958                    self.repository.id, hello.card.repo
1959                ),
1960                runtime,
1961            );
1962        }
1963        let verdict = self.trust_of_key(&hello.card.pk);
1964        if !verdict.is_trusted() {
1965            return self.refuse_in(
1966                RefusalCode::Untrusted,
1967                format!("the peer's key is not trusted here ({verdict:?}): discovery is not trust"),
1968                runtime,
1969            );
1970        }
1971        // The replay cache holds only nonces of trusted keys of this repository: a stranger's
1972        // hellos, however many, can neither fill it nor evict a real peer's nonce.
1973        {
1974            let mut table = self.table.lock().expect("cooperation table");
1975            let now = super::protocol::now();
1976            table
1977                .nonces
1978                .retain(|_, ts| now.saturating_sub(*ts) <= 2 * MAX_LINK_SKEW_SECONDS);
1979            let key = (hello.card.pk.clone(), hello.nonce.clone());
1980            if table.nonces.contains_key(&key) {
1981                drop(table);
1982                return self.refuse_in(
1983                    RefusalCode::Replay,
1984                    "this hello was already answered".into(),
1985                    runtime,
1986                );
1987            }
1988            if table.nonces.len() >= NONCE_CACHE {
1989                if let Some(oldest) = table
1990                    .nonces
1991                    .iter()
1992                    .min_by_key(|(_, ts)| **ts)
1993                    .map(|(k, _)| k.clone())
1994                {
1995                    table.nonces.remove(&oldest);
1996                }
1997            }
1998            table.nonces.insert(key, now);
1999        }
2000        let link = fresh_token();
2001        {
2002            let mut table = self.table.lock().expect("cooperation table");
2003            if !table.peers.contains_key(&runtime_key) && table.peers.len() >= MAX_PEERS {
2004                drop(table);
2005                return self.refuse_in(
2006                    RefusalCode::Capacity,
2007                    "the peer table is full".into(),
2008                    runtime,
2009                );
2010            }
2011            let restarts = &self.counters.restarts;
2012            let reconnects = &self.counters.reconnects;
2013            let peer = table
2014                .peers
2015                .entry(runtime_key.clone())
2016                .or_insert_with(|| PeerLink {
2017                    card: hello.card.clone(),
2018                    trust: verdict.clone(),
2019                    proto,
2020                    outbound: None,
2021                    inbound: None,
2022                    last_exchange: None,
2023                    linked_at: rfc3339_now(),
2024                    rtt: None,
2025                    failures: 0,
2026                    last_error: None,
2027                    handshakes: 0,
2028                    reconnects: 0,
2029                    restarts: 0,
2030                    expired: false,
2031                });
2032            if peer.card.instance != hello.card.instance {
2033                // A new run of the same runtime: its old links are dead on both sides.
2034                peer.restarts += 1;
2035                restarts.fetch_add(1, Ordering::Relaxed);
2036                peer.outbound = None;
2037                peer.linked_at = rfc3339_now();
2038            } else if peer.expired || (peer.handshakes > 0 && peer.inbound.is_some()) {
2039                peer.reconnects += 1;
2040                reconnects.fetch_add(1, Ordering::Relaxed);
2041            }
2042            if let Some(old) = peer.inbound.take() {
2043                table.inbound_links.remove(&old.link);
2044            }
2045            let peer = table.peers.get_mut(&runtime_key).expect("inserted above");
2046            peer.card = hello.card.clone();
2047            peer.trust = verdict;
2048            peer.proto = proto;
2049            peer.inbound = Some(Inbound {
2050                link: link.clone(),
2051                counter: 0,
2052            });
2053            peer.last_exchange = Some(Instant::now());
2054            peer.handshakes += 1;
2055            peer.expired = false;
2056            table
2057                .inbound_links
2058                .insert(link.clone(), runtime_key.clone());
2059            table.refused.remove(&format!("inbound:{runtime_key}"));
2060        }
2061        self.counters.handshakes_in.fetch_add(1, Ordering::Relaxed);
2062        tracing::info!(runtime_id = %self.own_key, peer = %runtime_key, link_id = %link, "mesh link accepted");
2063        let welcome = Welcome {
2064            proto,
2065            link,
2066            card: self.card.clone(),
2067            nonce: hello.nonce,
2068            ts: super::protocol::now(),
2069            marks: self.journal.marks(),
2070            heartbeat_seconds: self.config.heartbeat_seconds,
2071            expiry_seconds: self.config.expiry_seconds,
2072        };
2073        match serde_json::to_value(&welcome) {
2074            Ok(body) => LinkReply::accepted(sign(&self.identity, Domain::Welcome, body)),
2075            Err(e) => LinkReply::refused(RefusalCode::Malformed, e.to_string()),
2076        }
2077    }
2078
2079    /// Answer a sync round: the link must be known and its peer's instance current, the
2080    /// signature must be the peer's and the counter must rise; then ingest, merge, and
2081    /// answer with what the peer lacks.
2082    ///
2083    /// The rising counter is what makes a captured round worthless: a recorded request
2084    /// replayed later carries a counter that has already been used, and is refused without
2085    /// the events in it being looked at. Answering with what the peer lacks in the same
2086    /// round is what makes the exchange symmetrical — one call, both directions — so
2087    /// convergence does not depend on which side dials.
2088    ///
2089    /// ```
2090    /// # use std::collections::BTreeMap;
2091    /// # use std::sync::{Arc, Mutex, Weak};
2092    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2093    /// # use majordomus_cli::mesh::cooperation::*;
2094    /// # use majordomus_cli::mesh::identity::NodeIdentity;
2095    /// # use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
2096    /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, RefusalCode, Signed, HELLO_PATH};
2097    /// # use majordomus_cli::mesh::registry::MeshRegistry;
2098    /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2099    /// # use majordomus_cli::mesh::trust::TrustPolicy;
2100    /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2101    /// # impl LinkTransport for Net {
2102    /// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2103    /// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2104    /// #         .ok_or_else(|| format!("{e}: nobody there"))?;
2105    /// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2106    /// # let net = Arc::new(Net::default());
2107    /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2108    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2109    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2110    /// #     version: "doc".into(), config: CooperationConfig::default(),
2111    /// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2112    /// #     registry: Arc::new(MeshRegistry::new()),
2113    /// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2114    /// #     checkout: CheckoutFacts::default() }).unwrap();
2115    /// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2116    /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
2117    /// let key = a.dial(&["b:1".into()]).unwrap();
2118    /// b.claim(&SessionInfo::named("s1", "cli"), vec!["docs".into()], None,
2119    ///     ClaimMode::Advisory, None).unwrap();
2120    ///
2121    /// // A asks; B's answer carries B's own events back, so one round settles both sides
2122    /// a.sync_with(&key).unwrap();
2123    /// assert_eq!(a.state().claims.len(), 1, "B answered with what A lacked");
2124    /// assert_eq!(b.status().counters.syncs_in, 1);
2125    ///
2126    /// // a round under no link this runtime issued is refused before its events are read
2127    /// let stranger = Signed { body: serde_json::json!({"link": "ff", "counter": 1}), sig: "".into() };
2128    /// assert_eq!(b.accept_sync(&stranger).refusal.unwrap().code, RefusalCode::Malformed);
2129    /// ```
2130    pub fn accept_sync(&self, message: &Signed) -> LinkReply {
2131        if self.stopped() {
2132            return LinkReply::refused(RefusalCode::NotActive, "cooperation is stopped");
2133        }
2134        if serde_json::to_vec(message)
2135            .map(|b| b.len())
2136            .unwrap_or(usize::MAX)
2137            > MAX_LINK_MESSAGE
2138        {
2139            return self.refuse_in(
2140                RefusalCode::Oversized,
2141                "the sync exceeds the message bound".into(),
2142                None,
2143            );
2144        }
2145        let request: SyncRequest = match serde_json::from_value(message.body.clone()) {
2146            Ok(r) => r,
2147            Err(e) => {
2148                return self.refuse_in(
2149                    RefusalCode::Malformed,
2150                    format!("not a sync request: {e}"),
2151                    None,
2152                )
2153            }
2154        };
2155        let runtime_key = {
2156            let mut table = self.table.lock().expect("cooperation table");
2157            let Some(key) = table.inbound_links.get(&request.link).cloned() else {
2158                drop(table);
2159                Self::count_refusal(&self.counters.refused_in, RefusalCode::UnknownLink);
2160                return LinkReply::refused(
2161                    RefusalCode::UnknownLink,
2162                    "no such link here: say hello",
2163                );
2164            };
2165            let peer = table
2166                .peers
2167                .get_mut(&key)
2168                .expect("an indexed link has a peer");
2169            if peer.card.instance != request.instance {
2170                drop(table);
2171                Self::count_refusal(&self.counters.refused_in, RefusalCode::UnknownLink);
2172                return LinkReply::refused(
2173                    RefusalCode::UnknownLink,
2174                    "the link belongs to a previous instance: say hello",
2175                );
2176            }
2177            if !verify_signed(&peer.card.pk, Domain::SyncRequest, message) {
2178                drop(table);
2179                return self.refuse_in(
2180                    RefusalCode::Signature,
2181                    "the sync's signature is not the linked peer's".into(),
2182                    Some(key),
2183                );
2184            }
2185            let inbound = peer.inbound.as_mut().expect("an indexed link is inbound");
2186            if request.counter <= inbound.counter {
2187                drop(table);
2188                return self.refuse_in(
2189                    RefusalCode::Replay,
2190                    "the sync counter did not rise".into(),
2191                    Some(key),
2192                );
2193            }
2194            inbound.counter = request.counter;
2195            peer.last_exchange = Some(Instant::now());
2196            peer.expired = false;
2197            key
2198        };
2199        let report = self
2200            .journal
2201            .ingest(&request.events, &|e| self.origin_accept(e));
2202        self.journal.merge_marks(
2203            &request.marks,
2204            &|pk| self.trust_of_key(pk).is_trusted(),
2205            self.expiry(),
2206        );
2207        self.counters.syncs_in.fetch_add(1, Ordering::Relaxed);
2208        if report.accepted > 0 || report.rejected_total() > 0 {
2209            tracing::debug!(runtime_id = %self.own_key, peer = %runtime_key, accepted = report.accepted, duplicate = report.duplicate, rejected = report.rejected_total(), "mesh sync ingested");
2210        }
2211        let marks = self.journal.marks_for(&request.marks);
2212        let answer =
2213            self.sync_within_bound(Domain::SyncAnswer, &request.marks, |events| SyncAnswer {
2214                link: request.link.clone(),
2215                counter: request.counter,
2216                marks: marks.clone(),
2217                events,
2218                report: report.clone(),
2219            });
2220        match answer {
2221            Ok((signed, served)) => {
2222                self.counters
2223                    .events_served
2224                    .fetch_add(served as u64, Ordering::Relaxed);
2225                LinkReply::accepted(signed)
2226            }
2227            Err(e) => LinkReply::refused(RefusalCode::Oversized, e),
2228        }
2229    }
2230
2231    /// Sign a sync message carrying as many of the events `peer` lacks as fit: the bytes
2232    /// left under [`MAX_LINK_MESSAGE`] once everything else in the message is serialized,
2233    /// and never more than [`SYNC_EVENT_BUDGET`]. Answers the message and how many events it
2234    /// carries.
2235    ///
2236    /// The event budget was a constant that ignored what the marks weigh: a journal at its
2237    /// stream bounds sends marks of hundreds of kilobytes, and those with a full budget of
2238    /// events made a message the peer refuses as oversized, round after round, so a busy
2239    /// journal stopped replicating (I2139). The size is checked on the message as signed,
2240    /// where it is built, and an event that would take it over is left for the next round.
2241    fn sync_within_bound<T: serde::Serialize>(
2242        &self,
2243        domain: Domain,
2244        peer: &Marks,
2245        message: impl Fn(Vec<MeshEvent>) -> T,
2246    ) -> Result<(Signed, usize), String> {
2247        let signed = |events: Vec<MeshEvent>| -> Result<(Signed, usize), String> {
2248            let body = serde_json::to_value(message(events)).map_err(json_text)?;
2249            let signed = sign(&self.identity, domain, body);
2250            // a message that cannot be written out is no message under the bound
2251            let size = serde_json::to_vec(&signed).map_or(usize::MAX, |b| b.len());
2252            Ok((signed, size))
2253        };
2254        let (bare, overhead) = signed(Vec::new())?;
2255        if overhead > MAX_LINK_MESSAGE {
2256            return Err(format!(
2257                "a sync message without events is {overhead} bytes, over the {MAX_LINK_MESSAGE}-byte link bound"
2258            ));
2259        }
2260        let budget = (MAX_LINK_MESSAGE - overhead).min(SYNC_EVENT_BUDGET);
2261        let mut events = self.journal.missing_for(peer, budget);
2262        // compact JSON puts the events in the empty array the bare message carries, one
2263        // comma between each two: the size is known before it is serialized again
2264        let sizes: Vec<usize> = events
2265            .iter()
2266            .map(|e| serde_json::to_vec(e).map_or(MAX_LINK_MESSAGE, |b| b.len()))
2267            .collect();
2268        let mut total = overhead + sizes.iter().sum::<usize>() + sizes.len().saturating_sub(1);
2269        while total > MAX_LINK_MESSAGE && !events.is_empty() {
2270            events.pop();
2271            total -= sizes[events.len()] + usize::from(!events.is_empty());
2272        }
2273        if events.is_empty() {
2274            return Ok((bare, 0));
2275        }
2276        loop {
2277            let count = events.len();
2278            let (message, size) = signed(events.clone())?;
2279            if size <= MAX_LINK_MESSAGE {
2280                return Ok((message, count));
2281            }
2282            // the budget does not count the separators, and one event always goes: drop
2283            // from the end until the message fits; the rest follow next round
2284            events.pop();
2285            if events.is_empty() {
2286                return Ok((bare, 0));
2287            }
2288        }
2289    }
2290
2291    // ------------------------------------------------------------------ dialing
2292
2293    fn refuse_out(
2294        &self,
2295        endpoint: &str,
2296        refusal: LinkRefusal,
2297        runtime: Option<String>,
2298    ) -> RoundError {
2299        Self::count_refusal(&self.counters.refused_out, refusal.code);
2300        tracing::warn!(runtime_id = %self.own_key, endpoint = %endpoint, code = refusal.code.as_str(), detail = %refusal.detail, "mesh link not established");
2301        remember_refusal(
2302            &mut self.table.lock().expect("cooperation table"),
2303            endpoint.to_string(),
2304            Refused {
2305                refusal: refusal.clone(),
2306                runtime,
2307                direction: "outbound",
2308                at: rfc3339_now(),
2309            },
2310        );
2311        RoundError::Refused(refusal)
2312    }
2313
2314    /// Say hello at each endpoint in turn until one welcomes this runtime; verify the
2315    /// welcome as strictly as a hello is verified. Returns the peer's runtime key.
2316    ///
2317    /// A peer is dialed by its endpoints rather than by one address because a machine
2318    /// answers at several and only it knows which of them a caller can reach; the key that
2319    /// comes back is what the link is remembered by, so a peer that moves between
2320    /// addresses is still the same peer. Verifying the welcome as strictly as a hello is
2321    /// what keeps the trust decision symmetrical — dialing somebody is not a reason to
2322    /// believe what answers.
2323    ///
2324    /// ```
2325    /// # use std::collections::BTreeMap;
2326    /// # use std::sync::{Arc, Mutex, Weak};
2327    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2328    /// # use majordomus_cli::mesh::cooperation::*;
2329    /// # use majordomus_cli::mesh::identity::NodeIdentity;
2330    /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
2331    /// # use majordomus_cli::mesh::registry::MeshRegistry;
2332    /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2333    /// # use majordomus_cli::mesh::trust::TrustPolicy;
2334    /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2335    /// # impl LinkTransport for Net {
2336    /// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2337    /// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2338    /// #         .ok_or_else(|| format!("{e}: nobody there"))?;
2339    /// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2340    /// # let net = Arc::new(Net::default());
2341    /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2342    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2343    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2344    /// #     version: "doc".into(), config: CooperationConfig::default(),
2345    /// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2346    /// #     registry: Arc::new(MeshRegistry::new()),
2347    /// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2348    /// #     checkout: CheckoutFacts::default() }).unwrap();
2349    /// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2350    /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
2351    /// // the first address is dead, the second answers: one peer, found at the one that works
2352    /// let key = a.dial(&["nowhere:1".into(), "b:1".into()]).unwrap();
2353    /// assert_eq!(key, b.runtime_key());
2354    /// assert_eq!(a.peers()[0].endpoint.as_deref(), Some("b:1"));
2355    ///
2356    /// // dialing this runtime's own endpoint is a refusal, not a link to itself
2357    /// a.dial(&["a:1".into()]).expect_err("a runtime does not cooperate with itself");
2358    /// ```
2359    pub fn dial(&self, endpoints: &[String]) -> Result<String, RoundError> {
2360        self.dial_to(None, endpoints)
2361    }
2362
2363    /// [`Cooperation::dial`], addressed: when the dialer knows the runtime key it means to
2364    /// reach, the hello names it inside its signed bytes, and any other runtime that receives
2365    /// a copy refuses it as `misaddressed`.
2366    pub(crate) fn dial_to(
2367        &self,
2368        to: Option<&str>,
2369        endpoints: &[String],
2370    ) -> Result<String, RoundError> {
2371        let hello = Hello {
2372            proto_min: LINK_PROTOCOL_MIN,
2373            proto_max: LINK_PROTOCOL_MAX,
2374            card: self.card.clone(),
2375            nonce: fresh_token(),
2376            ts: super::protocol::now(),
2377            to: to.map(str::to_string),
2378        };
2379        let body =
2380            serde_json::to_value(&hello).map_err(|e| RoundError::Unreachable(e.to_string()))?;
2381        let message = sign(&self.identity, Domain::Hello, body);
2382        let mut last = "no endpoint".to_string();
2383        for endpoint in endpoints {
2384            let started = Instant::now();
2385            let reply = match self.transport.post(endpoint, HELLO_PATH, &message) {
2386                Ok(reply) => reply,
2387                Err(e) => {
2388                    last = e;
2389                    continue;
2390                }
2391            };
2392            if let Some(refusal) = reply.refusal {
2393                return Err(self.refuse_out(endpoint, refusal, None));
2394            }
2395            let Some(signed) = reply.signed else {
2396                return Err(self.refuse_out(
2397                    endpoint,
2398                    LinkRefusal::new(RefusalCode::Malformed, "an empty reply"),
2399                    None,
2400                ));
2401            };
2402            let welcome: Welcome = match serde_json::from_value(signed.body.clone()) {
2403                Ok(w) => w,
2404                Err(e) => {
2405                    return Err(self.refuse_out(
2406                        endpoint,
2407                        LinkRefusal::new(RefusalCode::Malformed, format!("not a welcome: {e}")),
2408                        None,
2409                    ))
2410                }
2411            };
2412            if let Err(e) = welcome.card.check() {
2413                return Err(self.refuse_out(
2414                    endpoint,
2415                    LinkRefusal::new(RefusalCode::Malformed, e),
2416                    None,
2417                ));
2418            }
2419            let runtime = welcome.card.runtime_key();
2420            let check = if !verify_signed(&welcome.card.pk, Domain::Welcome, &signed) {
2421                Some((
2422                    RefusalCode::Signature,
2423                    "the welcome's signature does not verify".to_string(),
2424                ))
2425            } else if welcome.nonce != hello.nonce {
2426                Some((
2427                    RefusalCode::Replay,
2428                    "the welcome answers another hello".into(),
2429                ))
2430            } else if welcome.ts.abs_diff(super::protocol::now()) > MAX_LINK_SKEW_SECONDS {
2431                Some((
2432                    RefusalCode::Stale,
2433                    "the welcome's clock is outside the skew window".into(),
2434                ))
2435            } else if negotiate(welcome.proto, welcome.proto).is_none() {
2436                Some((
2437                    RefusalCode::ProtocolUnsupported,
2438                    format!("the peer chose protocol {}", welcome.proto),
2439                ))
2440            } else if runtime.as_deref() == Some(self.own_key.as_str()) {
2441                Some((
2442                    RefusalCode::SelfLink,
2443                    "that endpoint is this runtime".into(),
2444                ))
2445            } else if welcome.card.repo != self.repository.id {
2446                Some((
2447                    RefusalCode::RepositoryMismatch,
2448                    format!("the peer serves repository {}", welcome.card.repo),
2449                ))
2450            } else if !self.trust_of_key(&welcome.card.pk).is_trusted() {
2451                Some((
2452                    RefusalCode::Untrusted,
2453                    "the peer's key is not trusted here: discovery is not trust".into(),
2454                ))
2455            } else {
2456                None
2457            };
2458            if let Some((code, detail)) = check {
2459                return Err(self.refuse_out(endpoint, LinkRefusal::new(code, detail), runtime));
2460            }
2461            let key = runtime.expect("a checked card has a runtime key");
2462            let verdict = self.trust_of_key(&welcome.card.pk);
2463            let mut table = self.table.lock().expect("cooperation table");
2464            if !table.peers.contains_key(&key) && table.peers.len() >= MAX_PEERS {
2465                drop(table);
2466                return Err(self.refuse_out(
2467                    endpoint,
2468                    LinkRefusal::new(RefusalCode::Capacity, "the peer table is full"),
2469                    Some(key),
2470                ));
2471            }
2472            let peer = table.peers.entry(key.clone()).or_insert_with(|| PeerLink {
2473                card: welcome.card.clone(),
2474                trust: verdict.clone(),
2475                proto: welcome.proto,
2476                outbound: None,
2477                inbound: None,
2478                last_exchange: None,
2479                linked_at: rfc3339_now(),
2480                rtt: None,
2481                failures: 0,
2482                last_error: None,
2483                handshakes: 0,
2484                reconnects: 0,
2485                restarts: 0,
2486                expired: false,
2487            });
2488            if peer.card.instance != welcome.card.instance {
2489                peer.restarts += 1;
2490                self.counters.restarts.fetch_add(1, Ordering::Relaxed);
2491                peer.linked_at = rfc3339_now();
2492            } else if peer.handshakes > 0 {
2493                peer.reconnects += 1;
2494                self.counters.reconnects.fetch_add(1, Ordering::Relaxed);
2495            }
2496            peer.card = welcome.card.clone();
2497            peer.trust = verdict;
2498            peer.proto = welcome.proto;
2499            peer.outbound = Some(Outbound {
2500                link: welcome.link.clone(),
2501                endpoint: endpoint.clone(),
2502                counter: 0,
2503                remote_marks: welcome.marks.clone(),
2504            });
2505            peer.last_exchange = Some(Instant::now());
2506            peer.rtt = Some(started.elapsed());
2507            peer.failures = 0;
2508            peer.last_error = None;
2509            peer.handshakes += 1;
2510            peer.expired = false;
2511            table.refused.remove(endpoint);
2512            drop(table);
2513            self.journal.merge_marks(
2514                &welcome.marks,
2515                &|pk| self.trust_of_key(pk).is_trusted(),
2516                self.expiry(),
2517            );
2518            self.counters.handshakes_out.fetch_add(1, Ordering::Relaxed);
2519            tracing::info!(runtime_id = %self.own_key, peer = %key, endpoint = %endpoint, link_id = %welcome.link, "mesh link established");
2520            return Ok(key);
2521        }
2522        self.counters.dial_failures.fetch_add(1, Ordering::Relaxed);
2523        Err(RoundError::Unreachable(last))
2524    }
2525
2526    /// One sync round with a peer this runtime dials: push what the peer lacks, take what
2527    /// this runtime lacks, in one exchange.
2528    ///
2529    /// Only the dialing side runs rounds, so a link has one driver and one counter however
2530    /// many threads hold it; an inbound-only peer is synced by its own side. A round is
2531    /// bounded by an event budget rather than by time, so a runtime that has been away for
2532    /// a day catches up over several rounds instead of one that never finishes.
2533    ///
2534    /// ```
2535    /// # use std::collections::BTreeMap;
2536    /// # use std::sync::{Arc, Mutex, Weak};
2537    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2538    /// # use majordomus_cli::mesh::cooperation::*;
2539    /// # use majordomus_cli::mesh::identity::NodeIdentity;
2540    /// # use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
2541    /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
2542    /// # use majordomus_cli::mesh::registry::MeshRegistry;
2543    /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2544    /// # use majordomus_cli::mesh::trust::TrustPolicy;
2545    /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2546    /// # impl LinkTransport for Net {
2547    /// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2548    /// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2549    /// #         .ok_or_else(|| format!("{e}: nobody there"))?;
2550    /// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2551    /// # let net = Arc::new(Net::default());
2552    /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2553    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2554    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2555    /// #     version: "doc".into(), config: CooperationConfig::default(),
2556    /// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2557    /// #     registry: Arc::new(MeshRegistry::new()),
2558    /// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2559    /// #     checkout: CheckoutFacts::default() }).unwrap();
2560    /// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2561    /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
2562    /// let key = a.dial(&["b:1".into()]).unwrap();
2563    /// a.claim(&SessionInfo::named("s1", "cli"), vec!["apps".into()], None,
2564    ///     ClaimMode::Exclusive, None).unwrap();
2565    ///
2566    /// for _ in 0..2 {
2567    ///     a.journal().beat_own();
2568    ///     a.sync_with(&key).unwrap();
2569    /// }
2570    /// assert_eq!(a.state().digest, b.state().digest, "the two now hold one state");
2571    ///
2572    /// // B's own claims come back over the same rounds, without B dialing anybody
2573    /// b.claim(&SessionInfo::named("s2", "cli"), vec!["docs".into()], None,
2574    ///     ClaimMode::Exclusive, None).unwrap();
2575    /// b.journal().beat_own();
2576    /// a.sync_with(&key).unwrap();
2577    /// assert_eq!(a.state().claims.len(), 2);
2578    /// ```
2579    pub fn sync_with(&self, runtime_key: &str) -> Result<(), RoundError> {
2580        // One round per link at a time: a verification and a worker (or two workers for one
2581        // peer) racing on the counter would refuse each other's rounds as replays.
2582        let round = {
2583            let mut rounds = self.rounds.lock().expect("sync rounds");
2584            Arc::clone(rounds.entry(runtime_key.to_string()).or_default())
2585        };
2586        let _round = round.lock().expect("sync round");
2587        let (out, pk) = {
2588            let mut table = self.table.lock().expect("cooperation table");
2589            let Some(peer) = table.peers.get_mut(runtime_key) else {
2590                return Err(RoundError::NoLink);
2591            };
2592            let Some(out) = peer.outbound.as_mut() else {
2593                return Err(RoundError::NoLink);
2594            };
2595            out.counter += 1;
2596            (out.clone(), peer.card.pk.clone())
2597        };
2598        let marks = self.journal.marks_for(&out.remote_marks);
2599        let (message, sent) = self
2600            .sync_within_bound(Domain::SyncRequest, &out.remote_marks, |events| {
2601                SyncRequest {
2602                    link: out.link.clone(),
2603                    counter: out.counter,
2604                    instance: self.card.instance.clone(),
2605                    marks: marks.clone(),
2606                    events,
2607                }
2608            })
2609            .map_err(RoundError::Unreachable)?;
2610        let sent = sent as u64;
2611        // The push is counted as the request leaves, not when its answer returns. The peer
2612        // holds these events from the moment it ingests them, before its answer is written,
2613        // so a count taken on the answer trails every reading of the peer: in between, the
2614        // events are held there and not yet sent here, and the count that lands a moment
2615        // later reads as an event that travelled twice. The peer counts what it serves the
2616        // same way, as it forms the answer. A round that fails after this has still put its
2617        // events on the wire, and a round that sends them again counts them again.
2618        self.counters.events_sent.fetch_add(sent, Ordering::Relaxed);
2619        let started = Instant::now();
2620        let failed = |why: String| {
2621            self.counters.syncs_failed.fetch_add(1, Ordering::Relaxed);
2622            let mut table = self.table.lock().expect("cooperation table");
2623            if let Some(peer) = table.peers.get_mut(runtime_key) {
2624                peer.failures += 1;
2625                peer.last_error = Some(why.clone());
2626            }
2627            why
2628        };
2629        let reply = match self.transport.post(&out.endpoint, SYNC_PATH, &message) {
2630            Ok(reply) => reply,
2631            Err(e) => return Err(RoundError::Unreachable(failed(e))),
2632        };
2633        if let Some(refusal) = reply.refusal {
2634            failed(refusal.to_string());
2635            if refusal.code == RefusalCode::UnknownLink {
2636                let mut table = self.table.lock().expect("cooperation table");
2637                if let Some(peer) = table.peers.get_mut(runtime_key) {
2638                    peer.outbound = None;
2639                }
2640            }
2641            Self::count_refusal(&self.counters.refused_out, refusal.code);
2642            return Err(RoundError::Refused(refusal));
2643        }
2644        let Some(signed) = reply.signed else {
2645            return Err(RoundError::Unreachable(failed(
2646                "an empty sync reply".into(),
2647            )));
2648        };
2649        let answer: SyncAnswer = match serde_json::from_value(signed.body.clone()) {
2650            Ok(a) => a,
2651            Err(e) => {
2652                return Err(RoundError::Unreachable(failed(format!(
2653                    "not a sync answer: {e}"
2654                ))))
2655            }
2656        };
2657        if !verify_signed(&pk, Domain::SyncAnswer, &signed)
2658            || answer.link != out.link
2659            || answer.counter != out.counter
2660        {
2661            return Err(RoundError::Refused(LinkRefusal::new(
2662                RefusalCode::Signature,
2663                failed("the sync answer is not the linked peer's answer to this round".into()),
2664            )));
2665        }
2666        self.journal
2667            .ingest(&answer.events, &|e| self.origin_accept(e));
2668        self.journal.merge_marks(
2669            &answer.marks,
2670            &|pk| self.trust_of_key(pk).is_trusted(),
2671            self.expiry(),
2672        );
2673        {
2674            let mut table = self.table.lock().expect("cooperation table");
2675            if let Some(peer) = table.peers.get_mut(runtime_key) {
2676                if let Some(o) = peer.outbound.as_mut() {
2677                    o.remote_marks = answer.marks;
2678                }
2679                peer.last_exchange = Some(Instant::now());
2680                peer.rtt = Some(started.elapsed());
2681                peer.failures = 0;
2682                peer.last_error = None;
2683                peer.expired = false;
2684            }
2685        }
2686        self.counters.syncs_out.fetch_add(1, Ordering::Relaxed);
2687        Ok(())
2688    }
2689
2690    // ------------------------------------------------------------------ supervision
2691
2692    fn targets(&self) -> Vec<Target> {
2693        let mut targets: Vec<Target> = self
2694            .config
2695            .seeds
2696            .iter()
2697            .map(|seed| Target {
2698                id: format!("seed:{seed}"),
2699                key: None,
2700                endpoints: vec![seed.trim_start_matches("http://").to_string()],
2701            })
2702            .collect();
2703        targets.extend(dial_targets(
2704            &self.registry.list(),
2705            &self.repository.id,
2706            &self.own_key,
2707            |public_key| self.trust_of_key(public_key).is_trusted(),
2708        ));
2709        targets
2710    }
2711
2712    fn reconcile_workers(self: &Arc<Self>) {
2713        let targets = self.targets();
2714        let mut table = self.table.lock().expect("cooperation table");
2715        let wanted: Vec<String> = targets.iter().map(|t| t.id.clone()).collect();
2716        table.workers.retain(|id, flag| {
2717            let keep = wanted.contains(id);
2718            if !keep {
2719                flag.store(true, Ordering::SeqCst);
2720            }
2721            keep
2722        });
2723        for target in targets {
2724            if table.workers.contains_key(&target.id) {
2725                continue;
2726            }
2727            let flag = Arc::new(AtomicBool::new(false));
2728            table.workers.insert(target.id.clone(), Arc::clone(&flag));
2729            let me = Arc::clone(self);
2730            me.counters.threads.fetch_add(1, Ordering::SeqCst);
2731            let _ = std::thread::Builder::new()
2732                .name("majordomus-mesh-link".into())
2733                .spawn(move || {
2734                    me.work(target, flag);
2735                    me.counters.threads.fetch_sub(1, Ordering::SeqCst);
2736                });
2737        }
2738    }
2739
2740    /// Whether this runtime should dial a known peer: the lower runtime key dials, and
2741    /// the higher one dials only when nothing has arrived from the peer lately — so a
2742    /// pair keeps one link when both can reach each other, and two when only one can.
2743    fn should_dial(&self, key: &str) -> bool {
2744        if self.own_key.as_str() < key {
2745            return true;
2746        }
2747        let table = self.table.lock().expect("cooperation table");
2748        let quiet = 3 * self.heartbeat();
2749        !table.peers.get(key).is_some_and(|p| {
2750            p.inbound.is_some() && p.last_exchange.is_some_and(|t| t.elapsed() <= quiet)
2751        })
2752    }
2753
2754    fn outbound_key(&self, key: &str) -> bool {
2755        let table = self.table.lock().expect("cooperation table");
2756        table.peers.get(key).is_some_and(|p| p.outbound.is_some())
2757    }
2758
2759    fn work(&self, target: Target, flag: Arc<AtomicBool>) {
2760        let mut learned = target.key.clone();
2761        let mut failures: u32 = 0;
2762        // An answerer that restarted forgets the link and answers `unknown_link` once; the
2763        // dialer says hello again at once. One that keeps answering it is retried no faster
2764        // than a heartbeat, or a peer that welcomes and refuses every sync drives a busy loop.
2765        let mut unknown_in_a_row: u32 = 0;
2766        while !self.stopped() && !flag.load(Ordering::SeqCst) {
2767            let linked = learned
2768                .as_deref()
2769                .filter(|k| self.outbound_key(k))
2770                .map(str::to_string);
2771            let outcome = match linked {
2772                Some(key) => self.sync_with(&key),
2773                None => {
2774                    let may = target.key.as_deref().is_none_or(|k| self.should_dial(k));
2775                    if !may {
2776                        self.sleep(self.heartbeat());
2777                        continue;
2778                    }
2779                    // addressed whenever the runtime behind the endpoints is known
2780                    let to = learned.as_deref().or(target.key.as_deref());
2781                    match self.dial_to(to, &target.endpoints) {
2782                        Ok(key) => {
2783                            learned = Some(key.clone());
2784                            // A fresh link syncs at once: the peer should see this
2785                            // runtime's state now, not a heartbeat from now.
2786                            self.sync_with(&key)
2787                        }
2788                        Err(e) => Err(e),
2789                    }
2790                }
2791            };
2792            let delay = self.after_round(&outcome, &mut failures, &mut unknown_in_a_row);
2793            self.sleep(delay);
2794        }
2795    }
2796
2797    /// How long [`Self::work`] waits after a round that ended in `outcome`, with the counts of
2798    /// failed rounds and of `unknown_link` answers in a row carried from round to round.
2799    fn after_round(
2800        &self,
2801        outcome: &Result<(), RoundError>,
2802        failures: &mut u32,
2803        unknown_in_a_row: &mut u32,
2804    ) -> Duration {
2805        match outcome {
2806            Ok(()) => {
2807                *failures = 0;
2808                *unknown_in_a_row = 0;
2809                self.heartbeat()
2810            }
2811            Err(RoundError::Refused(r)) if r.code == RefusalCode::UnknownLink => {
2812                *failures = 0;
2813                *unknown_in_a_row = unknown_in_a_row.saturating_add(1);
2814                if *unknown_in_a_row == 1 {
2815                    Duration::ZERO
2816                } else {
2817                    self.heartbeat()
2818                }
2819            }
2820            Err(RoundError::Refused(_)) => {
2821                *failures = failures.saturating_add(3).min(6);
2822                self.backoff(*failures)
2823            }
2824            Err(_) => {
2825                *failures = failures.saturating_add(1).min(6);
2826                self.backoff(*failures)
2827            }
2828        }
2829    }
2830
2831    /// Heartbeat × 2^failures, never beyond the expiry: an unreachable peer is asked less
2832    /// and less, and still within one expiry of coming back.
2833    fn backoff(&self, failures: u32) -> Duration {
2834        let factor = 1u32 << failures.min(6);
2835        (self.heartbeat() * factor).min(self.expiry().max(self.heartbeat()))
2836    }
2837
2838    fn expire_links(&self) {
2839        let expiry = self.expiry();
2840        let mut table = self.table.lock().expect("cooperation table");
2841        let mut dropped_links = Vec::new();
2842        let mut gone = Vec::new();
2843        for (key, peer) in table.peers.iter_mut() {
2844            let age = peer.last_exchange.map(|t| t.elapsed());
2845            if age.is_some_and(|a| a > expiry) && !peer.expired {
2846                peer.expired = true;
2847                if let Some(inbound) = peer.inbound.take() {
2848                    dropped_links.push(inbound.link);
2849                }
2850                peer.outbound = None;
2851                self.counters.peers_expired.fetch_add(1, Ordering::Relaxed);
2852                tracing::warn!(runtime_id = %self.own_key, peer = %key, "mesh link expired: no exchange within the expiry");
2853            }
2854            if age.is_some_and(|a| a > expiry + PEER_RETENTION) {
2855                gone.push(key.clone());
2856            }
2857        }
2858        for link in dropped_links {
2859            table.inbound_links.remove(&link);
2860        }
2861        for key in gone {
2862            table.peers.remove(&key);
2863        }
2864    }
2865
2866    fn state_of(&self, peer: &PeerLink) -> LinkState {
2867        let Some(age) = peer.last_exchange.map(|t| t.elapsed()) else {
2868            return LinkState::Connecting;
2869        };
2870        let expiry = self.expiry();
2871        if peer.expired || age > expiry {
2872            LinkState::Expired
2873        } else if age <= 2 * self.heartbeat() + Duration::from_secs(1) {
2874            LinkState::Connected
2875        } else if age <= expiry / 2 {
2876            LinkState::Degraded
2877        } else {
2878            LinkState::Unreachable
2879        }
2880    }
2881
2882    // ------------------------------------------------------------------ reading
2883
2884    /// Every peer this runtime has a link to, including the ones that have expired and not
2885    /// yet been forgotten. An expired peer is kept for a retention window rather than
2886    /// dropped on the spot, because "the peer I had is gone" is the answer a person needs
2887    /// when a machine stops answering, and a list that silently shrinks tells them nothing.
2888    ///
2889    /// ```
2890    /// # use std::collections::BTreeMap;
2891    /// # use std::sync::{Arc, Mutex, Weak};
2892    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2893    /// # use majordomus_cli::mesh::cooperation::*;
2894    /// # use majordomus_cli::mesh::identity::NodeIdentity;
2895    /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
2896    /// # use majordomus_cli::mesh::registry::MeshRegistry;
2897    /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2898    /// # use majordomus_cli::mesh::trust::TrustPolicy;
2899    /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2900    /// # impl LinkTransport for Net {
2901    /// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2902    /// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2903    /// #         .ok_or_else(|| format!("{e}: nobody there"))?;
2904    /// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2905    /// # let net = Arc::new(Net::default());
2906    /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2907    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2908    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2909    /// #     version: "doc".into(), config: CooperationConfig::default(),
2910    /// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2911    /// #     registry: Arc::new(MeshRegistry::new()),
2912    /// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2913    /// #     checkout: CheckoutFacts::default() }).unwrap();
2914    /// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2915    /// # let (a, b, c) = (runtime("a:1"), runtime("b:1"), runtime("c:1"));
2916    /// a.dial(&["b:1".into()]).unwrap();
2917    /// a.dial(&["c:1".into()]).unwrap();
2918    ///
2919    /// let keys: Vec<String> = a.peers().into_iter().map(|p| p.runtime).collect();
2920    /// assert_eq!(keys.len(), 2, "one entry per runtime, whoever dialed whom");
2921    /// assert!(keys.iter().any(|k| k == b.runtime_key()), "B is one of them");
2922    /// assert!(keys.iter().any(|k| k == c.runtime_key()), "and C is the other");
2923    /// assert_eq!(b.peers().len(), 1, "B was dialed once and holds one peer");
2924    /// ```
2925    pub fn peers(&self) -> Vec<PeerView> {
2926        let table = self.table.lock().expect("cooperation table");
2927        let own_node = self.identity.public.node_id.as_str();
2928        table
2929            .peers
2930            .iter()
2931            .map(|(key, peer)| PeerView {
2932                runtime: key.clone(),
2933                node: key[..32].to_string(),
2934                instance: peer.card.instance.clone(),
2935                name: peer.card.name.clone(),
2936                version: peer.card.version.clone(),
2937                local: &key[..32] == own_node,
2938                trust: peer.trust.clone(),
2939                protocol: peer.proto,
2940                features: peer.card.features.clone(),
2941                endpoints: peer.card.endpoints.clone(),
2942                state: self.state_of(peer),
2943                outbound: peer.outbound.is_some(),
2944                inbound: peer.inbound.is_some(),
2945                endpoint: peer.outbound.as_ref().map(|o| o.endpoint.clone()),
2946                last_exchange_ms: peer.last_exchange.map(|t| t.elapsed().as_millis() as u64),
2947                rtt_ms: peer.rtt.map(|d| d.as_millis() as u64),
2948                failures: peer.failures,
2949                last_error: peer.last_error.clone(),
2950                handshakes: peer.handshakes,
2951                reconnects: peer.reconnects,
2952                restarts: peer.restarts,
2953                linked_at: peer.linked_at.clone(),
2954            })
2955            .collect()
2956    }
2957
2958    /// Everything a person or an agent can ask about this runtime, in one reading: what it
2959    /// is, what it speaks, who it is linked to, what it refused, and what it has done.
2960    ///
2961    /// It is one call rather than several because the parts are only meaningful together —
2962    /// a peer count means nothing without the refusals beside it — and because two calls
2963    /// would be two moments, and a mesh changes between them.
2964    ///
2965    /// ```
2966    /// # use std::sync::Arc;
2967    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2968    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
2969    /// # use majordomus_cli::mesh::identity::NodeIdentity;
2970    /// # use majordomus_cli::mesh::link::HttpTransport;
2971    /// # use majordomus_cli::mesh::registry::MeshRegistry;
2972    /// # use majordomus_cli::mesh::repository::of_root_commits;
2973    /// # let repository = of_root_commits(&["root".into()]);
2974    /// # let cooperation = Cooperation::new(CooperationSetup {
2975    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
2976    /// #     repository: repository.clone(), endpoints: vec!["127.0.0.1:9".into()],
2977    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
2978    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
2979    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
2980    /// # }).unwrap();
2981    /// let status = cooperation.status();
2982    /// assert!(status.active, "this process is cooperating");
2983    /// assert_eq!(status.repository.unwrap().id, repository.id, "what links are matched on");
2984    /// assert_eq!(status.endpoints, ["127.0.0.1:9"], "where a peer would dial it");
2985    /// assert!(status.started_at.is_some(), "and since when");
2986    /// ```
2987    pub fn status(&self) -> CooperationStatus {
2988        let refused = {
2989            let table = self.table.lock().expect("cooperation table");
2990            table
2991                .refused
2992                .iter()
2993                .map(|(endpoint, r)| RefusedView {
2994                    endpoint: endpoint.clone(),
2995                    direction: r.direction.into(),
2996                    runtime: r.runtime.clone(),
2997                    refusal: r.refusal.clone(),
2998                    at: r.at.clone(),
2999                })
3000                .collect()
3001        };
3002        let c = &self.counters;
3003        let load = |a: &AtomicU64| a.load(Ordering::Relaxed);
3004        CooperationStatus {
3005            active: !self.stopped() && self.config.enabled,
3006            reason: if self.stopped() {
3007                Some("stopped".into())
3008            } else if !self.config.enabled {
3009                Some("cooperation is disabled in the mesh declaration".into())
3010            } else {
3011                None
3012            },
3013            runtime: Some(self.own_key.clone()),
3014            stream: Some(self.journal.own_stream().clone()),
3015            repository: Some(self.repository.clone()),
3016            protocol: (LINK_PROTOCOL_MIN, LINK_PROTOCOL_MAX),
3017            features: self.card.features.clone(),
3018            heartbeat_seconds: self.config.heartbeat_seconds,
3019            expiry_seconds: self.config.expiry_seconds,
3020            endpoints: self.card.endpoints.clone(),
3021            seeds: self.config.seeds.clone(),
3022            peers: self.peers(),
3023            refused,
3024            counters: CooperationCounters {
3025                handshakes_out: load(&c.handshakes_out),
3026                handshakes_in: load(&c.handshakes_in),
3027                dial_failures: load(&c.dial_failures),
3028                syncs_out: load(&c.syncs_out),
3029                syncs_failed: load(&c.syncs_failed),
3030                syncs_in: load(&c.syncs_in),
3031                events_sent: load(&c.events_sent),
3032                events_served: load(&c.events_served),
3033                reconnects: load(&c.reconnects),
3034                restarts: load(&c.restarts),
3035                peers_expired: load(&c.peers_expired),
3036                claims_refused: load(&c.claims_refused),
3037                threads: c.threads.load(Ordering::SeqCst),
3038                refused_in: c.refused_in.lock().expect("counters").clone(),
3039                refused_out: c.refused_out.lock().expect("counters").clone(),
3040            },
3041            journal: self.journal.tallies(),
3042            started_at: Some(self.started_at.clone()),
3043        }
3044    }
3045
3046    /// The folded state, liveness judged on this runtime's clock now.
3047    ///
3048    /// It is computed on every call rather than kept, because the answer changes with the
3049    /// clock and not only with the events: nothing has to happen for a claim to expire, so
3050    /// a cached state would go quietly wrong while nothing was arriving. Two runtimes that
3051    /// hold the same events and agree about who is alive fold the same state, which is what
3052    /// the digest is for.
3053    ///
3054    /// ```
3055    /// # use std::sync::Arc;
3056    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3057    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3058    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3059    /// # use majordomus_cli::mesh::link::HttpTransport;
3060    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3061    /// # use majordomus_cli::mesh::repository::of_root_commits;
3062    /// # let cooperation = Cooperation::new(CooperationSetup {
3063    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3064    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3065    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3066    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3067    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3068    /// # }).unwrap();
3069    /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3070    ///
3071    /// let empty = cooperation.state();
3072    /// assert!(empty.claims.is_empty());
3073    ///
3074    /// let session = SessionInfo::named("s1", "cli");
3075    /// let claim = cooperation
3076    ///     .claim(&session, vec!["apps".into()], None, ClaimMode::Exclusive, None)
3077    ///     .unwrap();
3078    /// let taken = cooperation.state();
3079    /// assert!(taken.claims[0].state.is_live());
3080    /// assert_ne!(taken.digest, empty.digest, "the digest is what a peer is compared to");
3081    ///
3082    /// cooperation.release(&claim.key).unwrap();
3083    /// assert!(!cooperation.state().claims[0].state.is_live(), "released, not forgotten");
3084    /// ```
3085    pub fn state(&self) -> CooperationState {
3086        let live = self.journal.stream_liveness(self.expiry());
3087        fold(&self.journal.events(), &|s| {
3088            live.get(s)
3089                .map(|(l, _)| *l)
3090                .unwrap_or(StreamLiveness::Expired)
3091        })
3092    }
3093
3094    /// Every stream's liveness and the milliseconds since its beat rose — the layer claims
3095    /// actually follow, which is not the layer links live on.
3096    ///
3097    /// A runtime this one has no link to is alive here as long as its beat keeps arriving
3098    /// through somebody else, and a runtime with a healthy link is dead here the moment it
3099    /// stops beating. That is what makes exclusivity survive a relay and a crash alike, and
3100    /// it is why this is worth reading beside [`Cooperation::peers`] rather than instead
3101    /// of it.
3102    ///
3103    /// ```
3104    /// # use std::sync::Arc;
3105    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3106    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3107    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3108    /// # use majordomus_cli::mesh::link::HttpTransport;
3109    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3110    /// # use majordomus_cli::mesh::repository::of_root_commits;
3111    /// # let cooperation = Cooperation::new(CooperationSetup {
3112    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3113    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3114    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3115    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3116    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3117    /// # }).unwrap();
3118    /// use majordomus_cli::mesh::journal::StreamLiveness;
3119    ///
3120    /// cooperation.journal().beat_own();
3121    /// let streams = cooperation.streams();
3122    /// let (id, liveness, age) = &streams[0];
3123    ///
3124    /// assert_eq!(id, cooperation.journal().own_stream(), "a runtime hears itself first");
3125    /// assert_eq!(*liveness, StreamLiveness::Own, "never judged against its own clock");
3126    /// assert!(age.is_some(), "and it knows when it last beat");
3127    /// ```
3128    pub fn streams(&self) -> Vec<(StreamId, StreamLiveness, Option<u64>)> {
3129        self.journal
3130            .stream_liveness(self.expiry())
3131            .into_iter()
3132            .map(|(id, (l, age))| (id, l, age.map(|a| a.as_millis() as u64)))
3133            .collect()
3134    }
3135
3136    /// Verify cooperation now: the local checks, then one sync round with every peer this
3137    /// runtime dials, and whether both sides hold the same marks afterwards.
3138    ///
3139    /// It acts rather than reports. Everything else here answers from what happened to
3140    /// arrive; this goes and asks, because the failure it exists to catch is the one where
3141    /// every counter looks reasonable and no traffic is actually crossing. It is therefore
3142    /// safe to run and not free: it writes nothing, but it costs a round per peer.
3143    ///
3144    /// ```
3145    /// # use std::collections::BTreeMap;
3146    /// # use std::sync::{Arc, Mutex, Weak};
3147    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3148    /// # use majordomus_cli::mesh::cooperation::*;
3149    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3150    /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
3151    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3152    /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
3153    /// # use majordomus_cli::mesh::trust::TrustPolicy;
3154    /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
3155    /// # impl LinkTransport for Net {
3156    /// #   fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
3157    /// #     let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
3158    /// #         .ok_or_else(|| format!("{e}: nobody there"))?;
3159    /// #     Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
3160    /// # let net = Arc::new(Net::default());
3161    /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
3162    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
3163    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
3164    /// #     version: "doc".into(), config: CooperationConfig::default(),
3165    /// #     trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
3166    /// #     registry: Arc::new(MeshRegistry::new()),
3167    /// #     transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
3168    /// #     checkout: CheckoutFacts::default() }).unwrap();
3169    /// #   net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
3170    /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
3171    /// a.dial(&["b:1".into()]).unwrap();
3172    ///
3173    /// a.journal().beat_own();
3174    /// let report = a.verify();
3175    /// assert!(report.ok, "every check held and every round went through");
3176    /// assert_eq!(report.peers[0].converged, Some(true), "and the two hold the same marks");
3177    /// assert_eq!(report.digest, Some(b.state().digest), "which is one state, seen twice");
3178    /// ```
3179    pub fn verify(&self) -> MeshVerifyReport {
3180        let mut checks = Vec::new();
3181        let heartbeat = self.heartbeat();
3182        checks.push(MeshVerifyCheck {
3183            check: "cooperation".into(),
3184            ok: !self.stopped() && self.config.enabled,
3185            detail: format!(
3186                "runtime {} — heartbeat {}s, expiry {}s, link protocol {LINK_PROTOCOL_MIN}..{LINK_PROTOCOL_MAX}",
3187                self.own_key, self.config.heartbeat_seconds, self.config.expiry_seconds
3188            ),
3189            impact: None,
3190            remediation: None,
3191        });
3192        checks.push(MeshVerifyCheck {
3193            check: "repository".into(),
3194            ok: true,
3195            detail: format!(
3196                "{} from {:?}: {}",
3197                self.repository.id, self.repository.basis, self.repository.detail
3198            ),
3199            impact: None,
3200            remediation: None,
3201        });
3202        let loopback = super::address::loopback_only(&self.card.endpoints);
3203        checks.push(MeshVerifyCheck {
3204            check: "endpoints".into(),
3205            ok: true,
3206            detail: self.card.endpoints.join(", "),
3207            impact: loopback.then(|| {
3208                "only runtimes on this machine can dial this one; this runtime can still dial remote runtimes, and a link works in both directions once dialed".into()
3209            }),
3210            remediation: loopback.then(|| {
3211                "to be dialed from other machines, serve beyond loopback: majordomus serve --host 0.0.0.0 (or a deployment object), on a private network or an overlay".into()
3212            }),
3213        });
3214        let own_age = self
3215            .journal
3216            .stream_liveness(self.expiry())
3217            .get(self.journal.own_stream())
3218            .and_then(|(_, age)| *age);
3219        let beating = own_age.is_some_and(|a| a <= 2 * heartbeat + Duration::from_secs(1));
3220        checks.push(MeshVerifyCheck {
3221            check: "heartbeat".into(),
3222            ok: beating,
3223            detail: match own_age {
3224                Some(a) => format!("this runtime's beat rose {}ms ago", a.as_millis()),
3225                None => "this runtime has not beaten yet".into(),
3226            },
3227            impact: (!beating).then(|| "linked runtimes will expire this one and stop honouring its claims".into()),
3228            remediation: (!beating).then(|| "the cooperation thread is not running: restart the server (majordomus serve stop && majordomus serve ensure)".into()),
3229        });
3230        let tallies = self.journal.tallies();
3231        checks.push(MeshVerifyCheck {
3232            check: "journal".into(),
3233            ok: tallies.pending == 0,
3234            detail: format!(
3235                "{} events in {} streams; {} received, {} duplicates absorbed, {} rejected, {} pending, {} opaque",
3236                tallies.events, tallies.streams, tallies.received, tallies.duplicates, tallies.rejected, tallies.pending, tallies.opaque
3237            ),
3238            impact: (tallies.pending > 0).then(|| "events after a gap are held until the gap arrives; the state lags the stream they belong to".into()),
3239            remediation: (tallies.pending > 0).then(|| "a peer relays a partial stream: run mesh verify on the peers and compare their marks".into()),
3240        });
3241        let refused = self.status().refused;
3242        let suspicious: Vec<&RefusedView> = refused
3243            .iter()
3244            .filter(|r| {
3245                matches!(
3246                    r.refusal.code,
3247                    RefusalCode::Untrusted
3248                        | RefusalCode::ProtocolUnsupported
3249                        | RefusalCode::Signature
3250                )
3251            })
3252            .collect();
3253        checks.push(MeshVerifyCheck {
3254            check: "refusals".into(),
3255            // A runtime of this repository refused for trust, protocol or signature is a
3256            // misconfiguration or an attack, never a healthy mesh. A repository mismatch is
3257            // isolation working and does not fail the verification.
3258            ok: suspicious.is_empty(),
3259            detail: if refused.is_empty() {
3260                "no candidate refused".into()
3261            } else {
3262                refused
3263                    .iter()
3264                    .map(|r| format!("{} {}: {}", r.endpoint, r.refusal.code.as_str(), r.refusal.detail))
3265                    .collect::<Vec<_>>()
3266                    .join("; ")
3267            },
3268            impact: (!suspicious.is_empty()).then(|| {
3269                "a runtime of this repository may be refused for trust, protocol or signature: it cooperates with nobody here".into()
3270            }),
3271            remediation: (!suspicious.is_empty()).then(|| {
3272                "untrusted: add its key (majordomus mesh identity on that machine) to trust.allow; protocol: upgrade the older executable; signature: that peer's identity file changed or the traffic is tampered".into()
3273            }),
3274        });
3275
3276        let mut peers = Vec::new();
3277        for peer in self.peers() {
3278            if peer.outbound {
3279                let outcome = self.sync_with(&peer.runtime);
3280                let after = self.peers().into_iter().find(|p| p.runtime == peer.runtime);
3281                let remote = {
3282                    let table = self.table.lock().expect("cooperation table");
3283                    table
3284                        .peers
3285                        .get(&peer.runtime)
3286                        .and_then(|p| p.outbound.as_ref())
3287                        .map(|o| o.remote_marks.clone())
3288                };
3289                let converged = remote.map(|remote| {
3290                    let local = self.journal.marks();
3291                    local.keys().chain(remote.keys()).all(|s| {
3292                        local.get(s).map(|m| m.seq).unwrap_or(0)
3293                            == remote.get(s).map(|m| m.seq).unwrap_or(0)
3294                    })
3295                });
3296                peers.push(PeerVerdict {
3297                    runtime: peer.runtime.clone(),
3298                    state: after
3299                        .as_ref()
3300                        .map(|p| p.state)
3301                        .unwrap_or(LinkState::Expired),
3302                    dialed: true,
3303                    round_trip: Some(outcome.is_ok()),
3304                    rtt_ms: after.and_then(|p| p.rtt_ms),
3305                    converged,
3306                    detail: match outcome {
3307                        Ok(()) => "one sync round, signed both ways".into(),
3308                        Err(e) => e.to_string(),
3309                    },
3310                });
3311            } else {
3312                let fresh = peer.state == LinkState::Connected;
3313                peers.push(PeerVerdict {
3314                    runtime: peer.runtime.clone(),
3315                    state: peer.state,
3316                    dialed: false,
3317                    round_trip: None,
3318                    rtt_ms: peer.rtt_ms,
3319                    converged: None,
3320                    detail: match peer.last_exchange_ms {
3321                        Some(ms) if fresh => {
3322                            format!("the peer dials this runtime; last exchange {ms}ms ago")
3323                        }
3324                        Some(ms) => {
3325                            format!("the peer dials this runtime and has been silent {ms}ms")
3326                        }
3327                        None => "the peer has not exchanged anything yet".into(),
3328                    },
3329                });
3330            }
3331        }
3332        // Declared seeds say that peers exist: a runtime with seeds and no connected peer has
3333        // verified nothing, however clean its local checks are.
3334        let connected = peers
3335            .iter()
3336            .filter(|p| p.state == LinkState::Connected)
3337            .count();
3338        let expected = self.config.seeds.len();
3339        let starved = expected > 0 && connected == 0;
3340        checks.push(MeshVerifyCheck {
3341            check: "links".into(),
3342            ok: !starved,
3343            detail: format!(
3344                "{connected} connected of {} linked peer(s); {expected} seed(s) declared",
3345                peers.len()
3346            ),
3347            impact: starved.then(|| {
3348                "the seeds this repository declares cooperate with nobody here: no claim, handover or review crosses".into()
3349            }),
3350            remediation: starved.then(|| {
3351                "read the refusals above; an unreachable seed is a network or bind problem — is the peer served beyond loopback?".into()
3352            }),
3353        });
3354        // An expired peer is listed with its state and does not fail the verification: a
3355        // peer that shut down cleanly is gone, not broken. A round that fails does.
3356        let ok = checks.iter().all(|c| c.ok)
3357            && peers
3358                .iter()
3359                .all(|p| p.state == LinkState::Expired || p.round_trip != Some(false));
3360        MeshVerifyReport {
3361            ok,
3362            runtime: Some(self.own_key.clone()),
3363            checks,
3364            peers,
3365            digest: Some(self.state().digest),
3366        }
3367    }
3368
3369    // ------------------------------------------------------------------ operations
3370
3371    fn write(&self, body: EventBody, key: String) -> Result<Written, CooperationError> {
3372        let appended = self
3373            .journal
3374            .append_acknowledged(body)
3375            .map_err(|e| CooperationError::Invalid(e.to_string()))?;
3376        Ok(Written {
3377            key,
3378            event: appended.event.id(),
3379            lamport: appended.event.lamport,
3380            durability: appended.durability,
3381        })
3382    }
3383
3384    fn own(&self, local: &str) -> String {
3385        format!("{}/{local}", self.journal.own_stream())
3386    }
3387
3388    fn session_exists(&self, state: &CooperationState, session: &str) -> bool {
3389        let key = self.own(session);
3390        state.sessions.iter().any(|s| s.key == key)
3391    }
3392
3393    /// Open a session of this runtime, or say again what an open one is doing. One call
3394    /// serves both because a session is identified by its name and described by the rest:
3395    /// announcing a new intent under an existing name is an update every runtime folds the
3396    /// same way, and nothing has to be told apart from what came before.
3397    ///
3398    /// ```
3399    /// # use std::sync::Arc;
3400    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3401    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3402    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3403    /// # use majordomus_cli::mesh::link::HttpTransport;
3404    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3405    /// # use majordomus_cli::mesh::repository::of_root_commits;
3406    /// # let cooperation = Cooperation::new(CooperationSetup {
3407    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3408    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3409    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3410    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3411    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3412    /// # }).unwrap();
3413    /// use majordomus_cli::mesh::journal::SessionInfo;
3414    ///
3415    /// let opened = cooperation.open_session(SessionInfo::named("s1", "claude-code")).unwrap();
3416    /// assert!(opened.key.ends_with("/s1"), "qualified by the stream that wrote it");
3417    ///
3418    /// let with_intent = SessionInfo { intent: Some("documenting the mesh".into()),
3419    ///     ..SessionInfo::named("s1", "claude-code") };
3420    /// cooperation.open_session(with_intent).unwrap();
3421    ///
3422    /// let sessions = cooperation.state().sessions;
3423    /// assert_eq!(sessions.len(), 1, "said again, not opened twice");
3424    /// assert_eq!(sessions[0].info.intent.as_deref(), Some("documenting the mesh"));
3425    /// ```
3426    pub fn open_session(&self, info: SessionInfo) -> Result<Written, CooperationError> {
3427        let key = self.own(&info.session);
3428        self.write(EventBody::SessionOpened { info }, key)
3429    }
3430
3431    /// Close a session of this runtime; its claims end with it.
3432    ///
3433    /// Claims end with the session rather than having to be released one by one, because
3434    /// the commonest way for a worker to stop is not an orderly one: what a session held
3435    /// must be recoverable from the fact that it is over. Only this runtime may close its
3436    /// own sessions — a session on another runtime is closed by that runtime, or expires
3437    /// when it stops beating.
3438    ///
3439    /// ```
3440    /// # use std::sync::Arc;
3441    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3442    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3443    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3444    /// # use majordomus_cli::mesh::link::HttpTransport;
3445    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3446    /// # use majordomus_cli::mesh::repository::of_root_commits;
3447    /// # let cooperation = Cooperation::new(CooperationSetup {
3448    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3449    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3450    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3451    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3452    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3453    /// # }).unwrap();
3454    /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3455    /// use majordomus_cli::mesh::state::SessionState;
3456    ///
3457    /// let session = SessionInfo::named("s1", "cli");
3458    /// cooperation.claim(&session, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
3459    /// cooperation.close_session("s1").unwrap();
3460    ///
3461    /// let state = cooperation.state();
3462    /// assert_eq!(state.sessions[0].state, SessionState::Closed);
3463    /// assert!(!state.claims[0].state.is_live(), "nobody had to release it");
3464    ///
3465    /// // and a name this runtime never opened is not something it can close
3466    /// cooperation.close_session("s9").expect_err("no such session here");
3467    /// ```
3468    pub fn close_session(&self, session: &str) -> Result<Written, CooperationError> {
3469        let state = self.state();
3470        if !self.session_exists(&state, session) {
3471            return Err(CooperationError::NotFound(format!(
3472                "no session '{session}' on this runtime"
3473            )));
3474        }
3475        self.write(
3476            EventBody::SessionClosed {
3477                session: session.into(),
3478            },
3479            self.own(session),
3480        )
3481    }
3482
3483    /// Claim a scope for a session of this runtime. An exclusive claim that meets a live
3484    /// exclusive claim of another session — on any runtime this one has heard — is
3485    /// refused with the claims it meets; the session is opened first if it is new.
3486    ///
3487    /// This is where exclusion actually happens. Admission is checked against the folded
3488    /// state of every runtime this one has heard from, under a lock, so two callers here
3489    /// cannot both be told yes; and an advisory claim is never refused, because its purpose
3490    /// is to make two workers visible to each other rather than to stop either.
3491    ///
3492    /// ```
3493    /// # use std::sync::Arc;
3494    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3495    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3496    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3497    /// # use majordomus_cli::mesh::link::HttpTransport;
3498    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3499    /// # use majordomus_cli::mesh::repository::of_root_commits;
3500    /// # let cooperation = Cooperation::new(CooperationSetup {
3501    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3502    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3503    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3504    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3505    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3506    /// # }).unwrap();
3507    /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3508    ///
3509    /// let first = SessionInfo::named("s1", "cli");
3510    /// cooperation.claim(&first, vec!["apps/majordomus-cli".into()],
3511    ///     Some("the mesh module".into()), ClaimMode::Exclusive, Some("#184".into())).unwrap();
3512    ///
3513    /// // a scope under a held one is the same place, and another session is refused it
3514    /// let second = SessionInfo::named("s2", "cli");
3515    /// cooperation.claim(&second, vec!["apps/majordomus-cli/src".into()], None,
3516    ///     ClaimMode::Exclusive, None).expect_err("the scope is claimed");
3517    ///
3518    /// // announcing the same work advisedly is always allowed: it informs, it does not exclude
3519    /// cooperation.claim(&second, vec!["apps/majordomus-cli/src".into()], None,
3520    ///     ClaimMode::Advisory, None).unwrap();
3521    /// assert_eq!(cooperation.state().overlaps.len(), 1, "two workers, told about each other");
3522    /// ```
3523    pub fn claim(
3524        &self,
3525        session: &SessionInfo,
3526        scope: Vec<String>,
3527        intent: Option<String>,
3528        mode: ClaimMode,
3529        issue: Option<String>,
3530    ) -> Result<Written, CooperationError> {
3531        // Check and append under one lock: two callers of this runtime must not both pass
3532        // admission for one scope and leave the fold to name a conflict it could prevent.
3533        let _admission = self.admission.lock().expect("claim admission");
3534        let state = self.state();
3535        let conflicts = admission_conflicts(&state, &self.own(&session.session), &scope, mode);
3536        if !conflicts.is_empty() {
3537            self.counters.claims_refused.fetch_add(1, Ordering::Relaxed);
3538            let conflicts: Vec<ClaimView> = conflicts.into_iter().cloned().collect();
3539            tracing::info!(runtime_id = %self.own_key, session_id = %session.session, conflicts = conflicts.len(), "mesh claim refused: the scope is claimed");
3540            return Err(CooperationError::Conflict(conflicts));
3541        }
3542        if !self.session_exists(&state, &session.session) {
3543            self.open_session(session.clone())?;
3544        }
3545        let claim = format!("c-{}", &fresh_token()[..12]);
3546        self.write(
3547            EventBody::ClaimAcquired {
3548                claim: claim.clone(),
3549                session: session.session.clone(),
3550                scope,
3551                intent,
3552                mode,
3553                issue,
3554            },
3555            self.own(&claim),
3556        )
3557    }
3558
3559    /// Release a claim this runtime took in its current run.
3560    ///
3561    /// Only the holder releases: a claim is a statement by one runtime about what it is
3562    /// doing, and letting another runtime withdraw it would make the record say something
3563    /// its author never said. A dead holder is not a problem this needs to solve — its
3564    /// stream stops beating and its claims stop excluding on every runtime at once.
3565    ///
3566    /// ```
3567    /// # use std::sync::Arc;
3568    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3569    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3570    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3571    /// # use majordomus_cli::mesh::link::HttpTransport;
3572    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3573    /// # use majordomus_cli::mesh::repository::of_root_commits;
3574    /// # let cooperation = Cooperation::new(CooperationSetup {
3575    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3576    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3577    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3578    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3579    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3580    /// # }).unwrap();
3581    /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3582    ///
3583    /// let mine = SessionInfo::named("s1", "cli");
3584    /// let held = cooperation
3585    ///     .claim(&mine, vec!["apps".into()], None, ClaimMode::Exclusive, None)
3586    ///     .unwrap();
3587    /// cooperation.release(&held.key).unwrap();
3588    ///
3589    /// // the scope is free again, for anybody
3590    /// let other = SessionInfo::named("s2", "cli");
3591    /// cooperation.claim(&other, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
3592    ///
3593    /// // a claim of another runtime is not this one's to withdraw
3594    /// cooperation.release("somebody-0000000000000002/c-abc").expect_err("not its author");
3595    /// ```
3596    pub fn release(&self, claim_key: &str) -> Result<Written, CooperationError> {
3597        let own_prefix = format!("{}/", self.journal.own_stream());
3598        let Some(local) = claim_key.strip_prefix(&own_prefix) else {
3599            return Err(CooperationError::NotOwn(format!(
3600                "{claim_key} was not claimed by this runtime's current run; only its holder releases a claim, and a dead holder's claim expires on its own"
3601            )));
3602        };
3603        let state = self.state();
3604        if !state.claims.iter().any(|c| c.key == claim_key) {
3605            return Err(CooperationError::NotFound(format!("no claim {claim_key}")));
3606        }
3607        self.write(
3608            EventBody::ClaimReleased {
3609                claim: local.into(),
3610            },
3611            claim_key.into(),
3612        )
3613    }
3614
3615    /// Publish a handover for every linked runtime to pick up.
3616    ///
3617    /// A handover is identified by the digest of what it says, so publishing the same
3618    /// continuity twice — by a retry, or by two runtimes that both saw it — leaves one
3619    /// handover rather than a choice between duplicates for whoever comes to take it.
3620    ///
3621    /// ```
3622    /// # use std::sync::Arc;
3623    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3624    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3625    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3626    /// # use majordomus_cli::mesh::link::HttpTransport;
3627    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3628    /// # use majordomus_cli::mesh::repository::of_root_commits;
3629    /// # let cooperation = Cooperation::new(CooperationSetup {
3630    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3631    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3632    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3633    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3634    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3635    /// # }).unwrap();
3636    /// use majordomus_cli::mesh::journal::HandoverBody;
3637    ///
3638    /// let body = "# Objective\nfinish the mesh docs\n".to_string();
3639    /// let handover = HandoverBody { id: HandoverBody::digest_of(&body), task: None, issue: None,
3640    ///     milestone: None, branch: Some("feature/mesh".into()), head: None, created_at: None,
3641    ///     name: None, body };
3642    ///
3643    /// let written = cooperation.publish_handover(handover.clone()).unwrap();
3644    /// assert_eq!(written.key, handover.id, "addressed by what it says");
3645    ///
3646    /// cooperation.publish_handover(handover).unwrap();
3647    /// assert_eq!(cooperation.state().handovers.len(), 1, "published twice, and still one");
3648    /// ```
3649    pub fn publish_handover(&self, handover: HandoverBody) -> Result<Written, CooperationError> {
3650        let id = handover.id.clone();
3651        self.write(EventBody::HandoverPublished { handover }, id)
3652    }
3653
3654    /// Take a handover: return its body and record, for everyone, that this session took
3655    /// it. The record is the point. Two workers picking up the same continuity is the
3656    /// failure this prevents, and it can only be prevented by the taking being visible —
3657    /// so reading a handover and consuming it are one operation, and a second worker sees
3658    /// who was there before it decides.
3659    ///
3660    /// Taking one already taken is not refused, because a handover may legitimately be
3661    /// picked up again; the record simply names everybody who has.
3662    ///
3663    /// ```
3664    /// # use std::sync::Arc;
3665    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3666    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3667    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3668    /// # use majordomus_cli::mesh::link::HttpTransport;
3669    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3670    /// # use majordomus_cli::mesh::repository::of_root_commits;
3671    /// # let cooperation = Cooperation::new(CooperationSetup {
3672    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3673    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3674    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3675    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3676    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3677    /// # }).unwrap();
3678    /// use majordomus_cli::mesh::journal::HandoverBody;
3679    ///
3680    /// let body = "# Objective\nfinish the mesh docs\n".to_string();
3681    /// let id = HandoverBody::digest_of(&body);
3682    /// cooperation.publish_handover(HandoverBody { id: id.clone(), task: None, issue: None,
3683    ///     milestone: None, branch: None, head: None, created_at: None, name: None, body }).unwrap();
3684    ///
3685    /// let (taken, _written) = cooperation.consume_handover(&id, "s1").unwrap();
3686    /// assert!(taken.handover.body.contains("mesh docs"), "the continuity itself");
3687    /// assert_eq!(cooperation.state().handovers[0].consumed_by.len(), 1, "and who took it");
3688    ///
3689    /// cooperation.consume_handover("no-such-handover", "s1").expect_err("nothing to take");
3690    /// ```
3691    pub fn consume_handover(
3692        &self,
3693        id: &str,
3694        session: &str,
3695    ) -> Result<(HandoverView, Written), CooperationError> {
3696        let state = self.state();
3697        let Some(view) = state.handovers.into_iter().find(|h| h.id == id) else {
3698            return Err(CooperationError::NotFound(format!(
3699                "no handover {id} in this runtime's journal"
3700            )));
3701        };
3702        let written = self.write(
3703            EventBody::HandoverConsumed {
3704                handover: id.into(),
3705                session: session.into(),
3706            },
3707            id.into(),
3708        )?;
3709        Ok((view, written))
3710    }
3711
3712    /// Ask for a review; a named reviewer must be a linked peer carrying `reviews`.
3713    ///
3714    /// A request may name nobody, and then it is an offer to the whole mesh — which is the
3715    /// useful shape when a worker wants a second opinion and does not care whose. Naming a
3716    /// reviewer is checked here rather than left to fail silently later: a request
3717    /// addressed to a runtime that is not linked, or to one whose executable does not do
3718    /// reviews, would otherwise sit unanswered and look like indifference.
3719    ///
3720    /// ```
3721    /// # use std::sync::Arc;
3722    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3723    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3724    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3725    /// # use majordomus_cli::mesh::link::HttpTransport;
3726    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3727    /// # use majordomus_cli::mesh::repository::of_root_commits;
3728    /// # let cooperation = Cooperation::new(CooperationSetup {
3729    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3730    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3731    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3732    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3733    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3734    /// # }).unwrap();
3735    /// use majordomus_cli::mesh::state::ReviewState;
3736    ///
3737    /// cooperation.request_review("s1", "feature/mesh-cooperation".into(),
3738    ///     vec!["apps/majordomus-cli/src/mesh".into()], Some("#184".into()), None, None).unwrap();
3739    ///
3740    /// let review = &cooperation.state().reviews[0];
3741    /// assert_eq!(review.subject, "feature/mesh-cooperation");
3742    /// assert_eq!(review.state, ReviewState::Open, "asked of anybody, answered by nobody yet");
3743    ///
3744    /// // a reviewer nobody is linked to is refused now, not left waiting
3745    /// cooperation.request_review("s1", "feature/x".into(), vec![], None,
3746    ///     Some("somebody-0000000000000002".into()), None).expect_err("no such peer");
3747    /// ```
3748    pub fn request_review(
3749        &self,
3750        session: &str,
3751        subject: String,
3752        scope: Vec<String>,
3753        issue: Option<String>,
3754        reviewer: Option<String>,
3755        body: Option<String>,
3756    ) -> Result<Written, CooperationError> {
3757        if let Some(target) = &reviewer {
3758            if *target != self.own_key {
3759                let table = self.table.lock().expect("cooperation table");
3760                match table.peers.get(target) {
3761                    None => {
3762                        return Err(CooperationError::NotFound(format!(
3763                            "no linked peer {target}"
3764                        )))
3765                    }
3766                    Some(peer) if !peer.card.supports("reviews") => {
3767                        return Err(CooperationError::FeatureUnsupported(format!(
3768                            "{target} does not carry the reviews feature (it carries: {})",
3769                            peer.card.features.join(", ")
3770                        )))
3771                    }
3772                    Some(_) => {}
3773                }
3774            }
3775        }
3776        let review = format!("r-{}", &fresh_token()[..12]);
3777        self.write(
3778            EventBody::ReviewRequested {
3779                review: review.clone(),
3780                session: session.into(),
3781                subject: redacted(subject),
3782                scope,
3783                issue,
3784                reviewer,
3785                body: body.map(redacted),
3786            },
3787            self.own(&review),
3788        )
3789    }
3790
3791    /// Answer a review request — anybody's, on any runtime, except the session that asked.
3792    ///
3793    /// The one refusal here is self-review, because a verdict from the session under review
3794    /// is not a second pair of eyes and a record that allowed it would make every review in
3795    /// the mesh worth less. Several sessions may answer one request and their verdicts all
3796    /// stand: the disagreement is the information.
3797    ///
3798    /// ```
3799    /// # use std::sync::Arc;
3800    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3801    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3802    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3803    /// # use majordomus_cli::mesh::link::HttpTransport;
3804    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3805    /// # use majordomus_cli::mesh::repository::of_root_commits;
3806    /// # let cooperation = Cooperation::new(CooperationSetup {
3807    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3808    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3809    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3810    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3811    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3812    /// # }).unwrap();
3813    /// use majordomus_cli::mesh::state::ReviewState;
3814    ///
3815    /// let asked = cooperation
3816    ///     .request_review("s1", "feature/mesh-cooperation".into(), vec![], None, None, None)
3817    ///     .unwrap();
3818    ///
3819    /// cooperation.answer_review(&asked.key, "s2", "approved".into(),
3820    ///     Some("the examples run".into())).unwrap();
3821    /// let review = &cooperation.state().reviews[0];
3822    /// assert_eq!(review.state, ReviewState::Answered);
3823    /// assert_eq!(review.answers[0].verdict, "approved");
3824    ///
3825    /// // the session under review does not get a vote on itself
3826    /// cooperation.answer_review(&asked.key, "s1", "approved".into(), None)
3827    ///     .expect_err("a review is another session's verdict");
3828    /// ```
3829    pub fn answer_review(
3830        &self,
3831        request: &str,
3832        session: &str,
3833        verdict: String,
3834        note: Option<String>,
3835    ) -> Result<Written, CooperationError> {
3836        let state = self.state();
3837        let Some(review) = state.reviews.iter().find(|r| r.key == request) else {
3838            return Err(CooperationError::NotFound(format!(
3839                "no review request {request}"
3840            )));
3841        };
3842        if review.session == self.own(session) {
3843            return Err(CooperationError::Invalid(format!(
3844                "session {session} requested review {request} and cannot answer it: a review is another session's verdict"
3845            )));
3846        }
3847        self.write(
3848            EventBody::ReviewAnswered {
3849                request: request.into(),
3850                session: session.into(),
3851                verdict: redacted(verdict),
3852                note: note.map(redacted),
3853            },
3854            request.into(),
3855        )
3856    }
3857
3858    /// End a review request this runtime asked for: done, void or superseded, with an
3859    /// optional result the request then carries. Any session of the asking runtime may —
3860    /// an MCP session is one connection, and the asker that reconnects is another session
3861    /// of the same runtime — and no other runtime may: a withdrawal is the requester's own
3862    /// record, never a verdict, so it does not count as an answer.
3863    ///
3864    /// ```
3865    /// # use std::sync::Arc;
3866    /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3867    /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3868    /// # use majordomus_cli::mesh::identity::NodeIdentity;
3869    /// # use majordomus_cli::mesh::link::HttpTransport;
3870    /// # use majordomus_cli::mesh::registry::MeshRegistry;
3871    /// # use majordomus_cli::mesh::repository::of_root_commits;
3872    /// # let cooperation = Cooperation::new(CooperationSetup {
3873    /// #     identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3874    /// #     repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3875    /// #     version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3876    /// #     journal_path: None, registry: Arc::new(MeshRegistry::new()),
3877    /// #     transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3878    /// # }).unwrap();
3879    /// use majordomus_cli::mesh::state::ReviewState;
3880    ///
3881    /// let asked = cooperation
3882    ///     .request_review("s1", "pre-flight of #919".into(), vec![], None, None,
3883    ///         Some("structure job and own cases, hook environment unset".into()))
3884    ///     .unwrap();
3885    ///
3886    /// // a request of another runtime cannot be ended here
3887    /// cooperation.withdraw_review("other-0000000000000009/r-1", "s1", None)
3888    ///     .expect_err("not this runtime's request");
3889    ///
3890    /// cooperation.withdraw_review(&asked.key, "s1", Some("clean: 45 gates, 12 cases".into()))
3891    ///     .unwrap();
3892    /// let review = &cooperation.state().reviews[0];
3893    /// assert_eq!(review.state, ReviewState::Withdrawn);
3894    /// assert_eq!(review.result.as_deref(), Some("clean: 45 gates, 12 cases"));
3895    /// assert!(review.answers.is_empty(), "a result is not an answer");
3896    /// ```
3897    pub fn withdraw_review(
3898        &self,
3899        request: &str,
3900        session: &str,
3901        note: Option<String>,
3902    ) -> Result<Written, CooperationError> {
3903        let state = self.state();
3904        let Some(review) = state.reviews.iter().find(|r| r.key == request) else {
3905            return Err(CooperationError::NotFound(format!(
3906                "no review request {request}"
3907            )));
3908        };
3909        if !review
3910            .key
3911            .starts_with(&format!("{}/", self.journal.own_stream()))
3912        {
3913            return Err(CooperationError::Invalid(format!(
3914                "review {request} was asked by {}, and only the runtime that asked may withdraw it",
3915                review.session
3916            )));
3917        }
3918        self.write(
3919            EventBody::ReviewWithdrawn {
3920                request: request.into(),
3921                session: session.into(),
3922                note: note.map(redacted),
3923            },
3924            request.into(),
3925        )
3926    }
3927}
3928
3929/// Free text of a review as it is signed: every credential shape replaced by the marker that
3930/// names it, through the one rule set (`redaction::redact_secrets`), as a handover body is.
3931///
3932/// A review's subject, body, verdict and notes are typed by a session and replicated in clear
3933/// to every linked runtime, so a pasted credential would otherwise travel with them (I2708).
3934/// The entries of `.ai/repo/ci/mesh-event-fields.yaml` that say `redacted` name these fields.
3935fn redacted(text: String) -> String {
3936    crate::redaction::redact_secrets(&text).text
3937}
3938
3939/// A JSON error as the text a refusal or a round error carries.
3940fn json_text(error: serde_json::Error) -> String {
3941    error.to_string()
3942}
3943
3944#[cfg(test)]
3945mod tests {
3946    use super::*;
3947    use crate::mesh::repository::of_root_commits;
3948    use crate::mesh::trust::TrustPolicy;
3949    use std::sync::Weak;
3950
3951    /// Two or more runtimes in one process, linked through this transport: the protocol
3952    /// under test is the real one; only the socket is replaced. Real sockets are the
3953    /// integration tests' (tests/mesh_cooperation.rs).
3954    #[derive(Default)]
3955    struct InProcess {
3956        runtimes: Mutex<BTreeMap<String, Weak<Cooperation>>>,
3957        cut: Mutex<Vec<String>>,
3958        /// Run after a peer has answered a sync and before the answer reaches the dialer:
3959        /// the window a slow machine stretches, held still.
3960        answered: Mutex<Option<Answered>>,
3961    }
3962
3963    type Answered = Box<dyn Fn() + Send>;
3964
3965    /// The one trust definition the live path and the reload share: this machine's own key
3966    /// is trusted, and a value that is not a key is rejected before any policy is asked —
3967    /// whatever the policy, even one that trusts on first use.
3968    #[test]
3969    fn a_value_that_is_not_a_key_is_rejected_by_every_policy() {
3970        let registry = MeshRegistry::new();
3971        let own = NodeIdentity::ephemeral().unwrap().public.public_key.clone();
3972        for policy in [TrustPolicy::Tofu, TrustPolicy::DenyUnknown] {
3973            let config = TrustConfig {
3974                policy,
3975                allow: vec!["not-a-key".into()],
3976            };
3977            assert!(key_trust(&own, &config, &registry, &own).is_trusted());
3978            assert_eq!(
3979                key_trust(&own, &config, &registry, "not-a-key"),
3980                TrustState::Rejected("not a key".into())
3981            );
3982        }
3983    }
3984
3985    impl LinkTransport for InProcess {
3986        fn post(&self, endpoint: &str, path: &str, message: &Signed) -> Result<LinkReply, String> {
3987            if self.cut.lock().unwrap().iter().any(|c| c == endpoint) {
3988                return Err(format!("{endpoint}: partitioned"));
3989            }
3990            let target = self
3991                .runtimes
3992                .lock()
3993                .unwrap()
3994                .get(endpoint)
3995                .and_then(Weak::upgrade)
3996                .ok_or_else(|| format!("{endpoint}: connection refused"))?;
3997            let reply = match path {
3998                HELLO_PATH => target.accept_hello(message),
3999                SYNC_PATH => target.accept_sync(message),
4000                other => return Err(format!("no route {other}")),
4001            };
4002            if path == SYNC_PATH {
4003                if let Some(answered) = self.answered.lock().unwrap().as_ref() {
4004                    answered();
4005                }
4006            }
4007            Ok(reply)
4008        }
4009    }
4010
4011    fn runtime(
4012        net: &Arc<InProcess>,
4013        endpoint: &str,
4014        repo: &str,
4015        policy: TrustPolicy,
4016    ) -> Arc<Cooperation> {
4017        let identity = Arc::new(NodeIdentity::ephemeral().unwrap());
4018        let c = Cooperation::new(CooperationSetup {
4019            identity,
4020            runtime: crate::mesh::repository::runtime_id(endpoint),
4021            repository: of_root_commits(&[repo.into()]),
4022            endpoints: vec![endpoint.into()],
4023            version: "test".into(),
4024            config: CooperationConfig {
4025                heartbeat_seconds: 1,
4026                expiry_seconds: 3,
4027                ..CooperationConfig::default()
4028            },
4029            trust: TrustConfig {
4030                policy,
4031                allow: vec![],
4032            },
4033            journal_path: None,
4034            registry: Arc::new(MeshRegistry::new()),
4035            transport: Arc::clone(net) as Arc<dyn LinkTransport>,
4036            board: None,
4037            checkout: CheckoutFacts::default(),
4038        })
4039        .unwrap();
4040        net.runtimes
4041            .lock()
4042            .unwrap()
4043            .insert(endpoint.into(), Arc::downgrade(&c));
4044        c
4045    }
4046
4047    fn session(name: &str) -> SessionInfo {
4048        SessionInfo::named(name, "test")
4049    }
4050
4051    #[test]
4052    fn retrust_ends_a_link_this_runtime_dialed_as_well_as_one_it_answered() {
4053        let net = Arc::new(InProcess::default());
4054        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4055        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4056        b.dial(&["a:1".into()]).expect("linked under tofu");
4057        // B dialed: it holds A outbound only, and A's key loses its trust all the same
4058        let ended = b.retrust(TrustConfig::default());
4059        assert_eq!(ended, vec![a.runtime_key().to_string()]);
4060        let synced = b.sync_with(a.runtime_key());
4061        assert!(matches!(synced, Err(RoundError::NoLink)), "{synced:?}");
4062    }
4063
4064    /// A body whose serialization fails once it carries an event.
4065    struct RefusesEvents(usize);
4066
4067    impl Serialize for RefusesEvents {
4068        fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
4069            if self.0 > 0 {
4070                return Err(serde::ser::Error::custom("no events here"));
4071            }
4072            serializer.serialize_unit()
4073        }
4074    }
4075
4076    #[test]
4077    fn a_sync_message_over_the_bound_or_unwritable_is_an_error_and_a_heavy_one_drops_events() {
4078        let net = Arc::new(InProcess::default());
4079        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4080        for name in ["s1", "s2"] {
4081            a.open_session(session(name)).unwrap();
4082        }
4083        let nothing = Marks::new();
4084        let over = a.sync_within_bound(Domain::SyncRequest, &nothing, |_| {
4085            "x".repeat(MAX_LINK_MESSAGE)
4086        });
4087        assert!(over.unwrap_err().contains("link bound"), "no event fits");
4088        let refused = a.sync_within_bound(Domain::SyncRequest, &nothing, |events| {
4089            RefusesEvents(events.len())
4090        });
4091        assert_eq!(refused.unwrap_err(), "no events here");
4092        let bare = a.sync_within_bound(Domain::SyncRequest, &nothing, |_| RefusesEvents(1));
4093        assert_eq!(
4094            bare.unwrap_err(),
4095            "no events here",
4096            "not even the bare message"
4097        );
4098        // a message that grows faster than its events loses events from the end until it fits
4099        let half = MAX_LINK_MESSAGE / 2;
4100        let (_, kept) = a
4101            .sync_within_bound(Domain::SyncRequest, &nothing, |events| {
4102                "x".repeat(events.len() * half)
4103            })
4104            .unwrap();
4105        assert_eq!(kept, 1, "one event fits beside its padding");
4106        let (bare, kept) = a
4107            .sync_within_bound(Domain::SyncRequest, &nothing, |events| {
4108                "x".repeat(events.len() * MAX_LINK_MESSAGE)
4109            })
4110            .unwrap();
4111        assert_eq!(kept, 0, "not one event fits, and the bare message goes");
4112        assert_eq!(bare.body, serde_json::json!(""));
4113    }
4114
4115    #[test]
4116    fn a_sync_over_the_link_bound_is_unreachable_and_an_answer_over_it_is_refused() {
4117        let net = Arc::new(InProcess::default());
4118        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4119        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4120        let key = a.dial(&["b:1".into()]).unwrap();
4121        a.sync_with(&key).unwrap();
4122        b.open_session(session("s1")).unwrap();
4123        let link = {
4124            let mut table = a.table.lock().unwrap();
4125            let out = table
4126                .peers
4127                .get_mut(&key)
4128                .unwrap()
4129                .outbound
4130                .as_mut()
4131                .unwrap();
4132            std::mem::replace(&mut out.link, "x".repeat(MAX_LINK_MESSAGE))
4133        };
4134        let synced = a.sync_with(&key);
4135        assert!(
4136            matches!(synced, Err(RoundError::Unreachable(ref e)) if e.contains("link bound")),
4137            "{synced:?}"
4138        );
4139        // a request exactly at the bound whose answer, carrying B's marks, is over it
4140        let request = |link: String| SyncRequest {
4141            link,
4142            counter: 1_000_000,
4143            instance: a.card.instance.clone(),
4144            marks: Marks::new(),
4145            events: vec![],
4146        };
4147        let signed = |r: &SyncRequest| {
4148            sign(
4149                &a.identity,
4150                Domain::SyncRequest,
4151                serde_json::to_value(r).unwrap(),
4152            )
4153        };
4154        let base = serde_json::to_vec(&signed(&request(String::new())))
4155            .unwrap()
4156            .len();
4157        let long = "y".repeat(MAX_LINK_MESSAGE - base);
4158        let message = signed(&request(long.clone()));
4159        assert_eq!(
4160            serde_json::to_vec(&message).unwrap().len(),
4161            MAX_LINK_MESSAGE
4162        );
4163        {
4164            let mut table = b.table.lock().unwrap();
4165            let peer = table.inbound_links[&link].clone();
4166            table.inbound_links.insert(long, peer);
4167        }
4168        let reply = b.accept_sync(&message);
4169        assert_eq!(
4170            reply.refusal.map(|r| r.code),
4171            Some(RefusalCode::Oversized),
4172            "the answer cannot be built under the bound"
4173        );
4174    }
4175
4176    #[test]
4177    fn a_round_s_outcome_sets_the_pause_before_the_next() {
4178        let net = Arc::new(InProcess::default());
4179        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4180        let refusal = |code| {
4181            Err(RoundError::Refused(LinkRefusal {
4182                code,
4183                detail: "test".into(),
4184                proto_min: LINK_PROTOCOL_MIN,
4185                proto_max: LINK_PROTOCOL_MAX,
4186            }))
4187        };
4188        let (mut failures, mut unknown) = (0u32, 0u32);
4189        let beat = a.heartbeat();
4190        let unknown_link = refusal(RefusalCode::UnknownLink);
4191        assert_eq!(
4192            a.after_round(&unknown_link, &mut failures, &mut unknown),
4193            Duration::ZERO,
4194            "a restarted answerer is greeted again at once"
4195        );
4196        assert_eq!(
4197            a.after_round(&unknown_link, &mut failures, &mut unknown),
4198            beat,
4199            "and no faster than a heartbeat after that"
4200        );
4201        assert_eq!(unknown, 2);
4202        let replay = refusal(RefusalCode::Replay);
4203        assert_eq!(
4204            a.after_round(&replay, &mut failures, &mut unknown),
4205            a.backoff(3)
4206        );
4207        assert_eq!(
4208            a.after_round(&Err(RoundError::NoLink), &mut failures, &mut unknown),
4209            a.backoff(4)
4210        );
4211        assert_eq!(failures, 4);
4212        assert_eq!(a.after_round(&Ok(()), &mut failures, &mut unknown), beat);
4213        assert_eq!((failures, unknown), (0, 0));
4214    }
4215
4216    #[test]
4217    fn a_hello_links_both_ways_and_a_claim_crosses_the_link() {
4218        let net = Arc::new(InProcess::default());
4219        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4220        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4221        let key = a.dial(&["b:1".into()]).expect("B welcomes A");
4222        assert_eq!(key, b.runtime_key());
4223        assert_eq!(
4224            b.peers()[0].runtime,
4225            a.runtime_key(),
4226            "B holds A as its inbound peer"
4227        );
4228
4229        a.claim(
4230            &session("s1"),
4231            vec!["apps".into()],
4232            None,
4233            ClaimMode::Exclusive,
4234            Some("#184".into()),
4235        )
4236        .unwrap();
4237        a.sync_with(&key).unwrap();
4238        a.journal.beat_own();
4239        a.sync_with(&key).unwrap();
4240        let at_b = b.state();
4241        assert_eq!(at_b.claims.len(), 1, "B sees A's claim");
4242        let refused = b.claim(
4243            &session("s9"),
4244            vec!["apps/majordomus-cli".into()],
4245            None,
4246            ClaimMode::Exclusive,
4247            None,
4248        );
4249        assert!(
4250            matches!(refused, Err(CooperationError::Conflict(ref c)) if c.len() == 1),
4251            "{refused:?}"
4252        );
4253        assert_eq!(
4254            a.state().digest,
4255            b.state().digest,
4256            "one state, two runtimes"
4257        );
4258    }
4259
4260    #[test]
4261    fn a_withdrawn_key_loses_its_link_without_a_restart() {
4262        let net = Arc::new(InProcess::default());
4263        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4264        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4265        b.dial(&["a:1".into()]).expect("linked under tofu");
4266        assert_eq!(a.peers().len(), 1);
4267        // A's declaration now trusts nobody it does not list, and lists nobody
4268        let ended = a.retrust(TrustConfig::default());
4269        assert_eq!(ended, vec![b.runtime_key().to_string()], "B's link ended");
4270        // the link B already holds is gone on A's side: a sync over it is not served
4271        let synced = b.sync_with(a.runtime_key());
4272        assert!(
4273            matches!(synced, Err(RoundError::Refused(ref r)) if r.code == RefusalCode::UnknownLink),
4274            "{synced:?}"
4275        );
4276        // B's next sync meets a link A no longer knows, and its hello is refused
4277        let reply = b.dial(&["a:1".into()]);
4278        assert!(
4279            matches!(reply, Err(RoundError::Refused(ref r)) if r.code == RefusalCode::Untrusted),
4280            "{reply:?}"
4281        );
4282        // and a trust that admits B again lets it link again
4283        a.retrust(TrustConfig {
4284            policy: TrustPolicy::DenyUnknown,
4285            allow: vec![b.identity.public.public_key.clone()],
4286        });
4287        assert!(
4288            b.dial(&["a:1".into()]).is_ok(),
4289            "trusted again, linked again"
4290        );
4291    }
4292
4293    #[test]
4294    fn a_hello_addressed_to_one_runtime_is_refused_by_another() {
4295        let net = Arc::new(InProcess::default());
4296        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4297        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4298        let c = runtime(&net, "c:1", "root", TrustPolicy::Tofu);
4299        let addressed = |to: Option<String>| {
4300            let hello = Hello {
4301                proto_min: LINK_PROTOCOL_MIN,
4302                proto_max: LINK_PROTOCOL_MAX,
4303                card: a.card.clone(),
4304                nonce: fresh_token(),
4305                ts: crate::mesh::protocol::now(),
4306                to,
4307            };
4308            sign(
4309                &a.identity,
4310                Domain::Hello,
4311                serde_json::to_value(&hello).unwrap(),
4312            )
4313        };
4314        // a hello on its way to B, captured and replayed to C: refused, and C links nobody
4315        let to_b = addressed(Some(b.runtime_key().to_string()));
4316        let refusal = c.accept_hello(&to_b).refusal.expect("refused");
4317        assert_eq!(refusal.code, RefusalCode::Misaddressed);
4318        assert!(c.peers().is_empty(), "the replay changed no link of C");
4319        // the same hello is B's to accept
4320        assert!(
4321            b.accept_hello(&to_b).refusal.is_none(),
4322            "B accepts what was meant for it"
4323        );
4324        // a hello that names nobody is still accepted: a seed, a first contact, an older release
4325        assert!(c.accept_hello(&addressed(None)).refusal.is_none());
4326        // and the dialer addresses what it knows: a dial to B by key reaches B
4327        assert_eq!(
4328            a.dial_to(Some(b.runtime_key()), &["b:1".into()]).unwrap(),
4329            b.runtime_key()
4330        );
4331        let wrong = a.dial_to(Some(c.runtime_key()), &["b:1".into()]);
4332        assert!(
4333            matches!(wrong, Err(RoundError::Refused(ref r)) if r.code == RefusalCode::Misaddressed),
4334            "{wrong:?}"
4335        );
4336    }
4337
4338    #[test]
4339    fn isolation_version_trust_and_replay_are_typed_refusals() {
4340        let net = Arc::new(InProcess::default());
4341        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4342        let other_repo = runtime(&net, "x:1", "another-root", TrustPolicy::Tofu);
4343        let strict = runtime(&net, "s:1", "root", TrustPolicy::DenyUnknown);
4344
4345        let code = |r: Result<String, RoundError>| match r {
4346            Err(RoundError::Refused(refusal)) => refusal.code,
4347            other => panic!("expected a refusal, got {other:?}"),
4348        };
4349        assert_eq!(
4350            code(a.dial(&["x:1".into()])),
4351            RefusalCode::RepositoryMismatch
4352        );
4353        assert!(
4354            other_repo.peers().is_empty(),
4355            "no cooperative relationship formed"
4356        );
4357        assert_eq!(code(a.dial(&["s:1".into()])), RefusalCode::Untrusted);
4358        assert!(strict.peers().is_empty());
4359        assert_eq!(code(a.dial(&["a:1".into()])), RefusalCode::SelfLink);
4360
4361        // A hello from a future protocol range, properly signed.
4362        let future = Hello {
4363            proto_min: LINK_PROTOCOL_MAX + 1,
4364            proto_max: LINK_PROTOCOL_MAX + 5,
4365            card: a.card.clone(),
4366            nonce: fresh_token(),
4367            ts: crate::mesh::protocol::now(),
4368            to: None,
4369        };
4370        let signed = sign(
4371            &a.identity,
4372            Domain::Hello,
4373            serde_json::to_value(&future).unwrap(),
4374        );
4375        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4376        let reply = b.accept_hello(&signed);
4377        let refusal = reply.refusal.expect("refused");
4378        assert_eq!(refusal.code, RefusalCode::ProtocolUnsupported);
4379        assert_eq!(
4380            (refusal.proto_min, refusal.proto_max),
4381            (LINK_PROTOCOL_MIN, LINK_PROTOCOL_MAX)
4382        );
4383
4384        // A replayed hello. (A fresh nonce first: the refused future hello's nonce is
4385        // already spent, which is exactly the replay rule.)
4386        let hello = Hello {
4387            proto_min: 1,
4388            proto_max: 1,
4389            nonce: fresh_token(),
4390            ..future
4391        };
4392        let signed = sign(
4393            &a.identity,
4394            Domain::Hello,
4395            serde_json::to_value(&hello).unwrap(),
4396        );
4397        assert!(b.accept_hello(&signed).signed.is_some());
4398        assert_eq!(
4399            b.accept_hello(&signed).refusal.unwrap().code,
4400            RefusalCode::Replay
4401        );
4402
4403        // Garbage, a forged signature, and an oversized message.
4404        let garbage = Signed {
4405            body: serde_json::json!({"nope": true}),
4406            sig: "00".into(),
4407        };
4408        assert_eq!(
4409            b.accept_hello(&garbage).refusal.unwrap().code,
4410            RefusalCode::Malformed
4411        );
4412        let mut forged = sign(
4413            &a.identity,
4414            Domain::Hello,
4415            serde_json::to_value(&Hello {
4416                nonce: fresh_token(),
4417                ..hello.clone()
4418            })
4419            .unwrap(),
4420        );
4421        forged.body["card"]["name"] = serde_json::json!("impostor");
4422        assert_eq!(
4423            b.accept_hello(&forged).refusal.unwrap().code,
4424            RefusalCode::Signature
4425        );
4426        let huge = Signed {
4427            body: serde_json::json!({"x": "y".repeat(MAX_LINK_MESSAGE)}),
4428            sig: String::new(),
4429        };
4430        assert_eq!(
4431            b.accept_hello(&huge).refusal.unwrap().code,
4432            RefusalCode::Oversized
4433        );
4434        assert!(b.status().counters.refused_in.len() >= 4);
4435    }
4436
4437    #[test]
4438    fn a_sync_under_an_unknown_link_or_a_replayed_counter_is_refused() {
4439        let net = Arc::new(InProcess::default());
4440        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4441        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4442        let key = a.dial(&["b:1".into()]).unwrap();
4443        a.sync_with(&key).unwrap();
4444        let replay = {
4445            let table = a.table.lock().unwrap();
4446            let out = table.peers[&key].outbound.clone().unwrap();
4447            SyncRequest {
4448                link: out.link,
4449                counter: out.counter,
4450                instance: a.card.instance.clone(),
4451                marks: Marks::new(),
4452                events: vec![],
4453            }
4454        };
4455        let signed = sign(
4456            &a.identity,
4457            Domain::SyncRequest,
4458            serde_json::to_value(&replay).unwrap(),
4459        );
4460        assert_eq!(
4461            b.accept_sync(&signed).refusal.unwrap().code,
4462            RefusalCode::Replay
4463        );
4464        let unknown = SyncRequest {
4465            link: fresh_token(),
4466            counter: 99,
4467            ..replay
4468        };
4469        let signed = sign(
4470            &a.identity,
4471            Domain::SyncRequest,
4472            serde_json::to_value(&unknown).unwrap(),
4473        );
4474        assert_eq!(
4475            b.accept_sync(&signed).refusal.unwrap().code,
4476            RefusalCode::UnknownLink
4477        );
4478    }
4479
4480    #[test]
4481    fn three_runtimes_in_a_line_converge_without_loops_or_duplicates() {
4482        let net = Arc::new(InProcess::default());
4483        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4484        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4485        let c = runtime(&net, "c:1", "root", TrustPolicy::Tofu);
4486        let ab = a.dial(&["b:1".into()]).unwrap();
4487        let cb = c.dial(&["b:1".into()]).unwrap();
4488        a.claim(
4489            &session("sa"),
4490            vec!["apps".into()],
4491            None,
4492            ClaimMode::Exclusive,
4493            None,
4494        )
4495        .unwrap();
4496        c.claim(
4497            &session("sc"),
4498            vec!["docs".into()],
4499            None,
4500            ClaimMode::Exclusive,
4501            None,
4502        )
4503        .unwrap();
4504        for _ in 0..3 {
4505            for j in [&a, &b, &c] {
4506                j.journal.beat_own();
4507            }
4508            a.sync_with(&ab).unwrap();
4509            c.sync_with(&cb).unwrap();
4510        }
4511        let digests: Vec<String> = [&a, &b, &c].iter().map(|r| r.state().digest).collect();
4512        assert!(
4513            digests.windows(2).all(|w| w[0] == w[1]),
4514            "A, B and C converge: {digests:?}"
4515        );
4516        let state = c.state();
4517        assert_eq!(state.claims.len(), 2);
4518        assert_eq!(
4519            c.journal.events().len(),
4520            4,
4521            "two sessions and two claims, each once"
4522        );
4523        // More rounds carry nothing: replication stopped, no event loops back.
4524        let before = b.status().counters.events_served;
4525        a.sync_with(&ab).unwrap();
4526        c.sync_with(&cb).unwrap();
4527        assert_eq!(
4528            b.status().counters.events_served,
4529            before,
4530            "nothing left to serve"
4531        );
4532        assert_eq!(
4533            a.status().journal.duplicates + c.status().journal.duplicates,
4534            0
4535        );
4536    }
4537
4538    #[test]
4539    fn a_send_is_counted_before_the_peer_can_be_seen_holding_it() {
4540        // The three-runtime integration test reads the counters once every runtime holds
4541        // every event, and asserts that quiet rounds add nothing. On a slow runner that
4542        // reading fell between a peer ingesting a push and the push's answer reaching the
4543        // dialer, which counted the push only then: the send landed after the reading and
4544        // passed for an event travelling twice. The transport here stops in that window.
4545        let net = Arc::new(InProcess::default());
4546        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4547        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4548        let key = a.dial(&["b:1".into()]).unwrap();
4549        for (r, name, scope) in [(&a, "sa", "apps"), (&b, "sb", "docs")] {
4550            r.claim(
4551                &session(name),
4552                vec![scope.into()],
4553                None,
4554                ClaimMode::Exclusive,
4555                None,
4556            )
4557            .unwrap();
4558        }
4559        let seen: Arc<Mutex<Vec<(usize, u64, u64)>>> = Arc::default();
4560        let (dialer, peer, log) = (Arc::downgrade(&a), Arc::downgrade(&b), Arc::clone(&seen));
4561        *net.answered.lock().unwrap() = Some(Box::new(move || {
4562            let (Some(dialer), Some(peer)) = (dialer.upgrade(), peer.upgrade()) else {
4563                return;
4564            };
4565            let held = peer
4566                .journal
4567                .events()
4568                .iter()
4569                .filter(|e| &e.stream == dialer.journal.own_stream())
4570                .count();
4571            log.lock().unwrap().push((
4572                held,
4573                dialer.status().counters.events_sent,
4574                peer.status().counters.events_served,
4575            ));
4576        }));
4577        a.sync_with(&key).unwrap();
4578        a.sync_with(&key).unwrap();
4579        assert_eq!(
4580            *seen.lock().unwrap(),
4581            vec![(2, 2, 2), (2, 2, 2)],
4582            "while the answer is on its way the peer holds the dialer's session and claim, \
4583             and both sides already count what they sent; the quiet round adds nothing"
4584        );
4585        assert_eq!(
4586            a.status().counters.events_sent,
4587            2,
4588            "and the answer adds nothing"
4589        );
4590        assert_eq!(
4591            a.status().journal.duplicates + b.status().journal.duplicates,
4592            0
4593        );
4594    }
4595
4596    #[test]
4597    fn a_partition_expires_the_far_side_and_healing_reconciles_to_one_winner() {
4598        let net = Arc::new(InProcess::default());
4599        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4600        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4601        let key = a.dial(&["b:1".into()]).unwrap();
4602        a.journal.beat_own();
4603        b.journal.beat_own();
4604        a.sync_with(&key).unwrap();
4605
4606        // Partition: both sides claim the same scope, as each is allowed to.
4607        net.cut.lock().unwrap().push("b:1".into());
4608        assert!(matches!(a.sync_with(&key), Err(RoundError::Unreachable(_))));
4609        assert!(a.peers()[0].failures >= 1);
4610        a.claim(
4611            &session("s1"),
4612            vec!["apps".into()],
4613            None,
4614            ClaimMode::Exclusive,
4615            None,
4616        )
4617        .unwrap();
4618        std::thread::sleep(Duration::from_millis(3200));
4619        b.claim(
4620            &session("s2"),
4621            vec!["apps".into()],
4622            None,
4623            ClaimMode::Exclusive,
4624            None,
4625        )
4626        .expect("A's stream has expired at B, so nothing there refuses B");
4627
4628        // Heal: one round each way, and both name the same single winner.
4629        net.cut.lock().unwrap().clear();
4630        a.journal.beat_own();
4631        b.journal.beat_own();
4632        match a.sync_with(&key) {
4633            Ok(()) => {}
4634            Err(RoundError::Refused(r)) if r.code == RefusalCode::UnknownLink => {
4635                let key = a.dial(&["b:1".into()]).unwrap();
4636                a.sync_with(&key).unwrap();
4637            }
4638            Err(e) => panic!("{e}"),
4639        }
4640        let (sa, sb) = (a.state(), b.state());
4641        assert_eq!(sa.digest, sb.digest, "the partition healed into one state");
4642        let held = sa
4643            .claims
4644            .iter()
4645            .filter(|c| c.state == crate::mesh::state::ClaimState::Held)
4646            .count();
4647        let conflicted = sa
4648            .claims
4649            .iter()
4650            .filter(|c| matches!(c.state, crate::mesh::state::ClaimState::Conflicted(_)))
4651            .count();
4652        assert_eq!(
4653            (held, conflicted),
4654            (1, 1),
4655            "one winner, one named conflict — never two holders"
4656        );
4657    }
4658
4659    #[test]
4660    fn a_reviewer_must_carry_the_feature_and_stop_ends_every_thread() {
4661        let net = Arc::new(InProcess::default());
4662        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4663        let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4664        let key = a.dial(&["b:1".into()]).unwrap();
4665        a.table
4666            .lock()
4667            .unwrap()
4668            .peers
4669            .get_mut(&key)
4670            .unwrap()
4671            .card
4672            .features = vec!["claims".into()];
4673        let refused = a.request_review(
4674            "s1",
4675            "feature/x".into(),
4676            vec![],
4677            None,
4678            Some(key.clone()),
4679            None,
4680        );
4681        assert!(
4682            matches!(refused, Err(CooperationError::FeatureUnsupported(_))),
4683            "{refused:?}"
4684        );
4685        assert!(matches!(
4686            a.request_review(
4687                "s1",
4688                "feature/x".into(),
4689                vec![],
4690                None,
4691                Some("nobody".into()),
4692                None
4693            ),
4694            Err(CooperationError::NotFound(_))
4695        ));
4696        assert!(a
4697            .request_review("s1", "feature/x".into(), vec![], None, None, None)
4698            .is_ok());
4699
4700        a.start();
4701        b.start();
4702        std::thread::sleep(Duration::from_millis(300));
4703        assert!(a.status().counters.threads >= 1);
4704        a.stop();
4705        b.stop();
4706        let deadline = Instant::now() + Duration::from_secs(3);
4707        while (a.status().counters.threads > 0 || b.status().counters.threads > 0)
4708            && Instant::now() < deadline
4709        {
4710            std::thread::sleep(Duration::from_millis(50));
4711        }
4712        assert_eq!(
4713            a.status().counters.threads,
4714            0,
4715            "stop leaves no cooperation thread behind"
4716        );
4717        assert_eq!(
4718            a.accept_hello(&Signed {
4719                body: serde_json::json!({}),
4720                sig: String::new()
4721            })
4722            .refusal
4723            .unwrap()
4724            .code,
4725            RefusalCode::NotActive
4726        );
4727    }
4728
4729    #[test]
4730    fn the_board_is_projected_into_sessions_and_advisory_claims_and_withdrawn_with_it() {
4731        let net = Arc::new(InProcess::default());
4732        let board = Arc::new(crate::peers::PeerBoard::new());
4733        let identity = Arc::new(NodeIdentity::ephemeral().unwrap());
4734        let a = Cooperation::new(CooperationSetup {
4735            identity,
4736            runtime: "00000000000000a1".into(),
4737            repository: of_root_commits(&["root".into()]),
4738            endpoints: vec!["a:1".into()],
4739            version: "test".into(),
4740            config: CooperationConfig::default(),
4741            trust: TrustConfig::default(),
4742            journal_path: None,
4743            registry: Arc::new(MeshRegistry::new()),
4744            transport: Arc::clone(&net) as Arc<dyn LinkTransport>,
4745            board: Some(Arc::clone(&board)),
4746            checkout: CheckoutFacts::default(),
4747        })
4748        .unwrap();
4749        let p = board.attach(crate::peers::Transport::Http);
4750        board.announce(&p, "implement the mesh", vec!["apps/majordomus-cli".into()]);
4751        a.project_board();
4752        a.project_board();
4753        let state = a.state();
4754        assert_eq!(state.sessions.len(), 1);
4755        assert_eq!(
4756            state.sessions[0].info.intent.as_deref(),
4757            Some("implement the mesh")
4758        );
4759        assert_eq!(state.claims.len(), 1, "projecting twice claims once");
4760        assert_eq!(state.claims[0].mode, ClaimMode::Advisory);
4761
4762        board.announce(&p, "write the docs", vec!["docs".into()]);
4763        a.project_board();
4764        let state = a.state();
4765        let live: Vec<_> = state.claims.iter().filter(|c| c.state.is_live()).collect();
4766        assert_eq!(live.len(), 1);
4767        assert_eq!(live[0].scope, vec!["docs".to_string()]);
4768
4769        // an intent past the journal's bound is projected once, clipped, and then left alone:
4770        // compared unclipped it differed from the projection on every pass and reopened the
4771        // session each time (986 of 1000 journal events on 2026-10-10)
4772        let long = format!("relay the train {}", "x".repeat(600));
4773        board.announce(&p, &long, vec!["docs".into()]);
4774        a.project_board();
4775        let once = a.state();
4776        assert_eq!(
4777            once.sessions[0].info.intent.as_deref().map(str::len),
4778            Some(512),
4779            "clipped to the bound"
4780        );
4781        let events = a.journal().events().len();
4782        a.project_board();
4783        a.project_board();
4784        assert_eq!(
4785            a.journal().events().len(),
4786            events,
4787            "an unchanged long intent writes nothing"
4788        );
4789        assert_eq!(a.state().digest, once.digest);
4790
4791        board.detach(&p);
4792        a.project_board();
4793        let state = a.state();
4794        assert!(
4795            state.claims.iter().all(|c| !c.state.is_live()),
4796            "a departed session holds nothing"
4797        );
4798    }
4799
4800    #[test]
4801    fn a_release_is_the_holders_and_a_session_close_ends_its_claims() {
4802        let net = Arc::new(InProcess::default());
4803        let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4804        let w = a
4805            .claim(
4806                &session("s1"),
4807                vec!["apps".into()],
4808                None,
4809                ClaimMode::Exclusive,
4810                None,
4811            )
4812            .unwrap();
4813        assert!(matches!(
4814            a.release("someone-else/c-1"),
4815            Err(CooperationError::NotOwn(_))
4816        ));
4817        a.release(&w.key).unwrap();
4818        let w2 = a
4819            .claim(
4820                &session("s1"),
4821                vec!["docs".into()],
4822                None,
4823                ClaimMode::Exclusive,
4824                None,
4825            )
4826            .unwrap();
4827        a.close_session("s1").unwrap();
4828        let state = a.state();
4829        let second = state.claims.iter().find(|c| c.key == w2.key).unwrap();
4830        assert!(matches!(
4831            second.state,
4832            crate::mesh::state::ClaimState::Expired(_)
4833        ));
4834    }
4835
4836    /// A registry record of node `node` (as a 32-hex node id) for the dial selection: the
4837    /// runtime slot `runtime`, one endpoint, repository `root`, public key `key`.
4838    fn record(node: usize, runtime: &str, presence: Presence, key: &str) -> NodeRecord {
4839        NodeRecord {
4840            node_id: serde_json::from_value(serde_json::json!(format!("{node:032x}"))).unwrap(),
4841            runtime: runtime.into(),
4842            public_key: key.into(),
4843            display_name: format!("node-{node}"),
4844            instance_id: serde_json::from_value(serde_json::json!("0000000000000001")).unwrap(),
4845            trust: TrustState::Observed,
4846            presence,
4847            endpoints: vec![format!("{runtime}:1")],
4848            capabilities: vec![],
4849            repositories: vec!["root".into()],
4850            protocol_version: 2,
4851            version: "test".into(),
4852            sources: vec![],
4853            first_seen: String::new(),
4854            last_seen: String::new(),
4855            restarts: 0,
4856        }
4857    }
4858
4859    fn ids(targets: &[Target]) -> Vec<String> {
4860        targets.iter().map(|t| t.id.clone()).collect()
4861    }
4862
4863    /// The registry keeps a stopped runtime's record for its retention, fifteen minutes, and
4864    /// every runtime of a machine lists the registry in one order. Twenty stopped worktree
4865    /// servers whose keys sort before the five that run filled the per-node cap with the
4866    /// dead: no live runtime dialed a live sibling, and the machine's worktrees shared no
4867    /// claims until the stopped records expired.
4868    #[test]
4869    fn a_node_s_present_runtimes_are_dialed_before_the_stopped_ones() {
4870        let mut records: Vec<NodeRecord> = (0..20)
4871            .map(|i| record(1, &format!("{i:016}"), Presence::Absent, "trusted"))
4872            .collect();
4873        records.extend(
4874            (100..105).map(|i| record(1, &format!("{i:016}"), Presence::Present, "trusted")),
4875        );
4876        let key = |r: &NodeRecord| format!("{}-{}", r.node_id, r.runtime);
4877        let live: Vec<String> = records[20..].iter().map(key).collect();
4878        let own = live[0].clone();
4879
4880        let chosen = ids(&dial_targets(&records, "root", &own, |k| k == "trusted"));
4881
4882        assert_eq!(chosen.len(), MAX_TARGETS_PER_NODE, "the cap still holds");
4883        assert!(!chosen.contains(&own), "a runtime never dials itself");
4884        let siblings: Vec<&String> = live[1..].iter().filter(|k| chosen.contains(k)).collect();
4885        assert_eq!(
4886            siblings.len(),
4887            live.len() - 1,
4888            "every live sibling is dialed; chosen: {chosen:?}"
4889        );
4890    }
4891
4892    use proptest::strategy::{Just, Strategy};
4893
4894    proptest::proptest! {
4895        /// For any registry: of a node with present runtimes, min(cap, present) of them are
4896        /// dialed; of every node, min(cap, eligible); and the choice does not depend on the
4897        /// order the records arrive in.
4898        #[test]
4899        fn the_dial_selection_takes_present_runtimes_first_whatever_the_order(
4900            (mix, order) in proptest::collection::vec(
4901                (0usize..3, proptest::bool::ANY, 0usize..4),
4902                0..48,
4903            )
4904            .prop_flat_map(|mix| {
4905                let order: Vec<usize> = (0..mix.len()).collect();
4906                (Just(mix), Just(order).prop_shuffle())
4907            })
4908        ) {
4909            let records: Vec<NodeRecord> = mix
4910                .iter()
4911                .enumerate()
4912                .map(|(i, (node, present, kind))| {
4913                    let presence = if *present { Presence::Present } else { Presence::Absent };
4914                    let key = if *kind == 0 { "stranger" } else { "trusted" };
4915                    record(*node, &format!("{i:016x}"), presence, key)
4916                })
4917                .collect();
4918            let shuffled: Vec<NodeRecord> = order.iter().map(|&i| records[i].clone()).collect();
4919            let own = format!("{:032x}-{:016x}", 0, 0);
4920            let trusted = |k: &str| k == "trusted";
4921
4922            let chosen = ids(&dial_targets(&records, "root", &own, trusted));
4923            proptest::prop_assert_eq!(
4924                &chosen,
4925                &ids(&dial_targets(&shuffled, "root", &own, trusted)),
4926                "the same records in another order choose the same targets"
4927            );
4928            for node in 0..3 {
4929                let prefix = format!("{node:032x}-");
4930                let eligible: Vec<(String, bool)> = records
4931                    .iter()
4932                    .filter(|r| r.node_id.as_str() == format!("{node:032x}"))
4933                    .filter(|r| trusted(&r.public_key))
4934                    .map(|r| {
4935                        let key = format!("{}-{}", r.node_id, r.runtime);
4936                        (key, r.presence == Presence::Present)
4937                    })
4938                    .filter(|(k, _)| *k != own)
4939                    .collect();
4940                let present = eligible.iter().filter(|(_, p)| *p).count();
4941                let of_node: Vec<&String> =
4942                    chosen.iter().filter(|k| k.starts_with(&prefix)).collect();
4943                let present_chosen = of_node
4944                    .iter()
4945                    .filter(|k| eligible.iter().any(|(e, p)| *p && e == **k))
4946                    .count();
4947                let cap = MAX_TARGETS_PER_NODE;
4948                proptest::prop_assert_eq!(of_node.len(), eligible.len().min(cap));
4949                proptest::prop_assert_eq!(present_chosen, present.min(cap));
4950            }
4951        }
4952    }
4953}