majordomus_cli/mesh/cooperation.rs
1//! Cooperation: the runtime that turns discovered, trusted runtimes of one repository into
2//! linked peers and keeps their journal converged. It owns the one peer-link table, the
3//! one journal of this runtime, the link handlers the HTTP surface answers, the dialers,
4//! the heartbeat and the expiry — and the operations (sessions, claims, handovers,
5//! reviews) every surface performs through it.
6//!
7//! The sequence one link goes through, and where each step is decided:
8//!
9//! ```text
10//! candidate a trusted registry record of the same repository, or a declared seed
11//! ↓
12//! dial hello → welcome (link::) signature, freshness, nonce, protocol,
13//! repository, self, trust — both directions
14//! ↓
15//! connected sync every heartbeat: marks + missing events, both ways, signed
16//! ↓ a failed round: degraded → unreachable (backoff, capped at the expiry)
17//! expired no exchange within the expiry: the link is dropped, the runtime's
18//! ↓ streams stop beating everywhere, its claims stop excluding
19//! reconnect the dialer says hello again; an unknown link id or a new instance is a
20//! re-handshake, never a second peer — the table is keyed by runtime
21//! ```
22//!
23//! StreamLiveness has two layers and they are deliberately separate. A *link* is this runtime's
24//! connection to one peer (connected, degraded, unreachable, expired). A *stream* is
25//! alive while its beat keeps rising anywhere in the mesh — a runtime linked only through
26//! a relay is still alive, and a runtime that crashed stops beating for everyone at once.
27//! Claims follow streams, not links.
28//!
29//! The whole of it, two runtimes in one process. Only the socket is replaced: the
30//! handshake, the signatures, the admission and the fold are the ones a LAN takes.
31//!
32//! ```
33//! use std::collections::BTreeMap;
34//! use std::sync::{Arc, Mutex, Weak};
35//!
36//! use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
37//! use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
38//! use majordomus_cli::mesh::identity::NodeIdentity;
39//! use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
40//! use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
41//! use majordomus_cli::mesh::registry::MeshRegistry;
42//! use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
43//! use majordomus_cli::mesh::trust::TrustPolicy;
44//!
45//! #[derive(Default)]
46//! struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
47//!
48//! impl LinkTransport for Net {
49//! fn post(&self, endpoint: &str, path: &str, m: &Signed) -> Result<LinkReply, String> {
50//! let peer = (self.0.lock().unwrap().get(endpoint).and_then(Weak::upgrade))
51//! .ok_or_else(|| format!("{endpoint}: nobody there"))?;
52//! Ok(if path == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) })
53//! }
54//! }
55//!
56//! let net = Arc::new(Net::default());
57//! let runtime = |endpoint: &str| {
58//! let c = Cooperation::new(CooperationSetup {
59//! identity: Arc::new(NodeIdentity::ephemeral().unwrap()),
60//! runtime: runtime_id(endpoint),
61//! repository: of_root_commits(&["root".into()]),
62//! endpoints: vec![endpoint.into()],
63//! version: "doc".into(),
64//! config: CooperationConfig::default(),
65//! trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] },
66//! journal_path: None,
67//! registry: Arc::new(MeshRegistry::new()),
68//! transport: Arc::clone(&net) as Arc<dyn LinkTransport>,
69//! board: None,
70//! checkout: CheckoutFacts::default(),
71//! })
72//! .unwrap();
73//! net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c));
74//! c
75//! };
76//!
77//! let (a, b) = (runtime("a:1"), runtime("b:1"));
78//! let key = a.dial(&["b:1".into()]).expect("B welcomes A");
79//! assert_eq!(key, b.runtime_key(), "the link is keyed by the peer's runtime");
80//!
81//! a.claim(&SessionInfo::named("s1", "cli"), vec!["apps".into()], None,
82//! ClaimMode::Exclusive, None).unwrap();
83//! for _ in 0..2 {
84//! a.journal().beat_own();
85//! a.sync_with(&key).unwrap();
86//! }
87//! assert_eq!(b.state().claims.len(), 1, "the claim crossed the link");
88//! assert_eq!(a.state().digest, b.state().digest, "one state, two runtimes");
89//! ```
90
91use std::collections::BTreeMap;
92use std::path::PathBuf;
93use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
94use std::sync::{Arc, Mutex};
95use std::time::{Duration, Instant};
96
97use schemars::JsonSchema;
98use serde::{Deserialize, Serialize};
99
100use super::config::{CooperationConfig, TrustConfig};
101use super::identity::{node_id_of_key, NodeIdentity};
102use super::journal::{
103 ClaimMode, Durability, EventBody, HandoverBody, Journal, JournalTallies, Marks, MeshEvent,
104 Rejection, SessionInfo, StreamId, StreamLiveness, HANDOVER_RETENTION,
105};
106use super::link::{
107 fresh_token, negotiate, sign, verify_signed, Domain, Hello, LinkRefusal, LinkReply,
108 LinkTransport, RefusalCode, RuntimeCard, Signed, SyncAnswer, SyncRequest, Welcome, FEATURES,
109 HELLO_PATH, LINK_PROTOCOL_MAX, LINK_PROTOCOL_MIN, MAX_LINK_MESSAGE, MAX_LINK_SKEW_SECONDS,
110 SYNC_EVENT_BUDGET, SYNC_PATH,
111};
112use super::registry::{MeshRegistry, NodeRecord, Presence};
113use super::repository::MeshRepositoryIdentity;
114use super::state::{admission_conflicts, fold, ClaimView, CooperationState, HandoverView};
115use super::trust::{self, TrustState};
116
117/// The most peers one runtime links to.
118pub const MAX_PEERS: usize = 256;
119
120/// How long an expired peer stays listed before it leaves the table.
121pub const PEER_RETENTION: Duration = Duration::from_secs(15 * 60);
122
123/// The most hello nonces remembered for replay refusal.
124const NONCE_CACHE: usize = 4096;
125
126/// The stop check's slice while sleeping.
127const SLICE: Duration = Duration::from_millis(100);
128
129/// Everything one runtime needs before it can link to another, gathered in one place so
130/// that the runtime never reaches for it later. Each field is a decision somebody else
131/// already made — the declaration's policy, the checkout's identity, this process's key,
132/// the socket to speak through — and passing them in is what lets a test replace any of
133/// them without a fixture: a fake transport, an ephemeral key, a journal that never
134/// touches the disk.
135///
136/// ```
137/// use std::sync::Arc;
138///
139/// use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
140/// use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
141/// use majordomus_cli::mesh::identity::NodeIdentity;
142/// use majordomus_cli::mesh::link::HttpTransport;
143/// use majordomus_cli::mesh::registry::MeshRegistry;
144/// use majordomus_cli::mesh::repository::of_root_commits;
145///
146/// let setup = CooperationSetup {
147/// identity: Arc::new(NodeIdentity::ephemeral().unwrap()),
148/// runtime: "0000000000000001".into(),
149/// repository: of_root_commits(&["root".into()]),
150/// endpoints: vec!["127.0.0.1:9".into()],
151/// version: "doc".into(),
152/// config: CooperationConfig::default(),
153/// trust: TrustConfig::default(),
154/// journal_path: None,
155/// registry: Arc::new(MeshRegistry::new()),
156/// transport: Arc::new(HttpTransport),
157/// board: None,
158/// checkout: CheckoutFacts::default(),
159/// };
160///
161/// let cooperation = Cooperation::new(setup).unwrap();
162/// assert!(cooperation.runtime_key().ends_with("-0000000000000001"));
163/// ```
164pub struct CooperationSetup {
165 /// This node's identity.
166 pub identity: Arc<NodeIdentity>,
167 /// This runtime's slot, 16 hex.
168 pub runtime: String,
169 /// The repository's mesh identity.
170 pub repository: MeshRepositoryIdentity,
171 /// Where this runtime answers.
172 pub endpoints: Vec<String>,
173 /// The executable version.
174 pub version: String,
175 /// The declaration's cooperation section.
176 pub config: CooperationConfig,
177 /// The declaration's trust section.
178 pub trust: TrustConfig,
179 /// Where the journal persists, when anywhere.
180 pub journal_path: Option<PathBuf>,
181 /// The discovery registry candidates come from.
182 pub registry: Arc<MeshRegistry>,
183 /// How link messages travel.
184 pub transport: Arc<dyn LinkTransport>,
185 /// The peer board of this process, projected into the journal every heartbeat: each
186 /// attached session becomes a mesh session and each of its announcements an advisory
187 /// claim, so a board announcement is visible on every linked runtime.
188 pub board: Option<Arc<crate::peers::PeerBoard>>,
189 /// Facts about this checkout the projected sessions carry.
190 pub checkout: CheckoutFacts,
191}
192
193/// What the projected sessions say about the checkout they work in. Computed once at
194/// start — branch and head are refreshed when a board session changes, never on a
195/// heartbeat, so liveness never rescans the repository.
196///
197/// Both fields are optional because a runtime need not be in a checkout at all, and a
198/// runtime that is says so once: the root is kept rather than the branch, so that a
199/// session announced after a checkout moved carries where it is now and not where it was
200/// when the process started.
201///
202/// ```
203/// use majordomus_cli::mesh::cooperation::CheckoutFacts;
204///
205/// let unknown = CheckoutFacts::default();
206/// assert!(unknown.root.is_none(), "a runtime outside a checkout claims no branch");
207///
208/// let here = CheckoutFacts { id: Some("c0ffee".into()), root: Some(".".into()) };
209/// assert_eq!(here.id.as_deref(), Some("c0ffee"));
210/// ```
211#[derive(Debug, Clone, Default)]
212pub struct CheckoutFacts {
213 /// The checkout id (a digest).
214 pub id: Option<String>,
215 /// The checkout root, for reading the branch and the head when a session changes.
216 pub root: Option<PathBuf>,
217}
218
219impl CheckoutFacts {
220 fn git(&self, args: &[&str]) -> Option<String> {
221 let root = self.root.as_ref()?;
222 let out = std::process::Command::new("git")
223 .arg("-C")
224 .arg(root)
225 .args(args)
226 .output()
227 .ok()?;
228 out.status
229 .success()
230 .then(|| String::from_utf8_lossy(&out.stdout).trim().to_string())
231 .filter(|s| !s.is_empty())
232 }
233}
234
235#[derive(Default)]
236struct Projected {
237 info: SessionInfo,
238 claims: BTreeMap<String, (String, Vec<String>, String)>,
239}
240
241/// A link's standing, from this runtime's side. It is a statement about a connection and
242/// never about the peer: a runtime reachable only through a third party is `Expired` here
243/// while its streams beat on everywhere, which is why claims follow streams and not this.
244///
245/// The variants are ordered by how far the link has fallen, so the worst link in a table
246/// is the maximum and a reader never has to rank them by hand.
247///
248/// ```
249/// use majordomus_cli::mesh::cooperation::LinkState;
250///
251/// let table = [LinkState::Connected, LinkState::Unreachable, LinkState::Degraded];
252/// assert_eq!(table.iter().max(), Some(&LinkState::Unreachable), "the worst link shows");
253/// assert_eq!(serde_json::to_value(LinkState::Connected).unwrap(), "connected");
254/// ```
255#[derive(
256 Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema,
257)]
258#[serde(rename_all = "snake_case")]
259pub enum LinkState {
260 /// Handshaking, or never yet exchanged.
261 Connecting,
262 /// An exchange within two heartbeats.
263 Connected,
264 /// Rounds are failing; the last exchange is under half the expiry old.
265 Degraded,
266 /// The last exchange is over half the expiry old; the link is about to expire.
267 Unreachable,
268 /// No exchange within the expiry: the link is dropped and will be re-dialed.
269 Expired,
270}
271
272#[derive(Clone)]
273struct Outbound {
274 link: String,
275 endpoint: String,
276 counter: u64,
277 remote_marks: Marks,
278}
279
280#[derive(Clone)]
281struct Inbound {
282 link: String,
283 counter: u64,
284}
285
286struct PeerLink {
287 card: RuntimeCard,
288 trust: TrustState,
289 proto: u32,
290 outbound: Option<Outbound>,
291 inbound: Option<Inbound>,
292 last_exchange: Option<Instant>,
293 linked_at: String,
294 rtt: Option<Duration>,
295 failures: u32,
296 last_error: Option<String>,
297 handshakes: u64,
298 reconnects: u64,
299 restarts: u64,
300 expired: bool,
301}
302
303struct Refused {
304 refusal: LinkRefusal,
305 runtime: Option<String>,
306 direction: &'static str,
307 at: String,
308}
309
310#[derive(Default)]
311struct Table {
312 peers: BTreeMap<String, PeerLink>,
313 inbound_links: BTreeMap<String, String>,
314 nonces: BTreeMap<(String, String), u64>,
315 refused: BTreeMap<String, Refused>,
316 workers: BTreeMap<String, Arc<AtomicBool>>,
317}
318
319#[derive(Default)]
320struct Counters {
321 handshakes_out: AtomicU64,
322 handshakes_in: AtomicU64,
323 dial_failures: AtomicU64,
324 syncs_out: AtomicU64,
325 syncs_failed: AtomicU64,
326 syncs_in: AtomicU64,
327 events_sent: AtomicU64,
328 events_served: AtomicU64,
329 reconnects: AtomicU64,
330 restarts: AtomicU64,
331 peers_expired: AtomicU64,
332 claims_refused: AtomicU64,
333 threads: AtomicU64,
334 refused_in: Mutex<BTreeMap<RefusalCode, u64>>,
335 refused_out: Mutex<BTreeMap<RefusalCode, u64>>,
336}
337
338/// One peer as every surface shows it: who it is, how this runtime reaches it, and what
339/// the link has been through. The counters are carried per peer rather than only in the
340/// totals because the question a person asks of a flapping mesh is which peer is flapping,
341/// and `restarts` answers a different one from `reconnects` — a peer whose process keeps
342/// dying, against a link that keeps dropping under a process that never did.
343///
344/// ```
345/// # use std::collections::BTreeMap;
346/// # use std::sync::{Arc, Mutex, Weak};
347/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
348/// # use majordomus_cli::mesh::cooperation::*;
349/// # use majordomus_cli::mesh::identity::NodeIdentity;
350/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
351/// # use majordomus_cli::mesh::registry::MeshRegistry;
352/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
353/// # use majordomus_cli::mesh::trust::TrustPolicy;
354/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
355/// # impl LinkTransport for Net {
356/// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
357/// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
358/// # .ok_or_else(|| format!("{e}: nobody there"))?;
359/// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
360/// # let net = Arc::new(Net::default());
361/// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
362/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
363/// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
364/// # version: "doc".into(), config: CooperationConfig::default(),
365/// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
366/// # registry: Arc::new(MeshRegistry::new()),
367/// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
368/// # checkout: CheckoutFacts::default() }).unwrap();
369/// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
370/// # let (a, b) = (runtime("a:1"), runtime("b:1"));
371/// a.dial(&["b:1".into()]).unwrap();
372///
373/// let peer: &PeerView = &a.peers()[0];
374/// assert_eq!(peer.runtime, b.runtime_key(), "the durable identity, not the endpoint");
375/// assert!(peer.outbound, "this runtime dials it");
376/// assert_eq!(peer.handshakes, 1);
377/// assert_eq!(peer.restarts, 0, "the peer's process has not died under the link");
378/// ```
379#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
380pub struct PeerView {
381 /// `<node>-<runtime>`: the peer's durable identity.
382 pub runtime: String,
383 /// The node (machine key digest).
384 pub node: String,
385 /// The current instance.
386 pub instance: String,
387 /// The node's display name.
388 pub name: String,
389 /// The executable version.
390 pub version: String,
391 /// Whether the peer runs on this machine (the same node key).
392 pub local: bool,
393 /// The trust verdict the link was admitted under.
394 pub trust: TrustState,
395 /// The negotiated link protocol.
396 pub protocol: u32,
397 /// The link features the peer carries.
398 pub features: Vec<String>,
399 /// Where the peer answers.
400 pub endpoints: Vec<String>,
401 /// The link's standing.
402 pub state: LinkState,
403 /// Whether this runtime dials the peer.
404 pub outbound: bool,
405 /// Whether the peer dials this runtime.
406 pub inbound: bool,
407 /// The endpoint this runtime dials, when it does.
408 #[serde(default, skip_serializing_if = "Option::is_none")]
409 pub endpoint: Option<String>,
410 /// Milliseconds since the last successful exchange in either direction.
411 #[serde(default, skip_serializing_if = "Option::is_none")]
412 pub last_exchange_ms: Option<u64>,
413 /// The last round trip, milliseconds.
414 #[serde(default, skip_serializing_if = "Option::is_none")]
415 pub rtt_ms: Option<u64>,
416 /// Consecutive failed rounds.
417 pub failures: u32,
418 /// The last failure, in words.
419 #[serde(default, skip_serializing_if = "Option::is_none")]
420 pub last_error: Option<String>,
421 /// Handshakes completed with this peer.
422 pub handshakes: u64,
423 /// Handshakes after an expiry or a dropped link.
424 pub reconnects: u64,
425 /// New instances of the peer seen (its process restarted).
426 pub restarts: u64,
427 /// When the current link was established, RFC 3339.
428 pub linked_at: String,
429}
430
431/// A candidate that was refused, with the rule that refused it. A refusal is kept and
432/// shown rather than logged and dropped, because the commonest mesh failure is a link that
433/// never forms: a person who declared a seed and sees no peer needs the decision that was
434/// made about it — the wrong repository, a key nobody allowed, a clock too far out — and
435/// the direction says which side made it.
436///
437/// ```
438/// # use std::collections::BTreeMap;
439/// # use std::sync::{Arc, Mutex, Weak};
440/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
441/// # use majordomus_cli::mesh::cooperation::*;
442/// # use majordomus_cli::mesh::identity::NodeIdentity;
443/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, RefusalCode, Signed, HELLO_PATH};
444/// # use majordomus_cli::mesh::registry::MeshRegistry;
445/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
446/// # use majordomus_cli::mesh::trust::TrustPolicy;
447/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
448/// # impl LinkTransport for Net {
449/// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
450/// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
451/// # .ok_or_else(|| format!("{e}: nobody there"))?;
452/// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
453/// # let net = Arc::new(Net::default());
454/// # let runtime = |endpoint: &str, policy: TrustPolicy| {
455/// # let c = Cooperation::new(CooperationSetup {
456/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
457/// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
458/// # version: "doc".into(), config: CooperationConfig::default(),
459/// # trust: TrustConfig { policy, allow: vec![] }, journal_path: None,
460/// # registry: Arc::new(MeshRegistry::new()),
461/// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
462/// # checkout: CheckoutFacts::default() }).unwrap();
463/// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
464/// # let a = runtime("a:1", TrustPolicy::Tofu);
465/// # let strict = runtime("s:1", TrustPolicy::DenyUnknown);
466/// a.dial(&["s:1".into()]).expect_err("reachable, and not trusted");
467///
468/// let refused: &RefusedView = &strict.status().refused[0];
469/// assert_eq!(refused.refusal.code, RefusalCode::Untrusted);
470/// assert_eq!(refused.direction, "inbound", "this runtime refused the hello");
471/// assert_eq!(refused.runtime.as_deref(), Some(a.runtime_key()));
472/// ```
473#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
474pub struct RefusedView {
475 /// The endpoint dialed, or `inbound:<runtime>` for a refused hello.
476 pub endpoint: String,
477 /// `outbound` (this runtime refused the peer's welcome, or the peer refused this
478 /// hello) or `inbound` (this runtime refused the peer's hello).
479 pub direction: String,
480 /// The peer's runtime key, when its card said.
481 #[serde(default, skip_serializing_if = "Option::is_none")]
482 pub runtime: Option<String>,
483 /// The refusal.
484 pub refusal: LinkRefusal,
485 /// When, RFC 3339.
486 pub at: String,
487}
488
489/// The link and replication counters since start: what this runtime did, not what it
490/// holds. They are monotonic and never reset, so two readings a minute apart are a rate —
491/// which is the only way to tell a mesh that is working from one that is retrying, since
492/// both show the same peers in the same states.
493///
494/// The counts are split by direction throughout (`handshakes_out` against
495/// `handshakes_in`, `refused_in` against `refused_out`) because a runtime that refuses
496/// everyone and one that everyone refuses need opposite remedies.
497///
498/// ```
499/// # use std::collections::BTreeMap;
500/// # use std::sync::{Arc, Mutex, Weak};
501/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
502/// # use majordomus_cli::mesh::cooperation::*;
503/// # use majordomus_cli::mesh::identity::NodeIdentity;
504/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
505/// # use majordomus_cli::mesh::registry::MeshRegistry;
506/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
507/// # use majordomus_cli::mesh::trust::TrustPolicy;
508/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
509/// # impl LinkTransport for Net {
510/// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
511/// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
512/// # .ok_or_else(|| format!("{e}: nobody there"))?;
513/// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
514/// # let net = Arc::new(Net::default());
515/// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
516/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
517/// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
518/// # version: "doc".into(), config: CooperationConfig::default(),
519/// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
520/// # registry: Arc::new(MeshRegistry::new()),
521/// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
522/// # checkout: CheckoutFacts::default() }).unwrap();
523/// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
524/// # let (a, b) = (runtime("a:1"), runtime("b:1"));
525/// let key = a.dial(&["b:1".into()]).unwrap();
526/// a.sync_with(&key).unwrap();
527///
528/// let dialer: CooperationCounters = a.status().counters;
529/// let answerer = b.status().counters;
530/// assert_eq!((dialer.handshakes_out, dialer.syncs_out), (1, 1));
531/// assert_eq!((answerer.handshakes_in, answerer.syncs_in), (1, 1), "the other side of it");
532/// assert_eq!(answerer.handshakes_out, 0, "B never dialed anybody");
533/// ```
534#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
535pub struct CooperationCounters {
536 /// Handshakes this runtime completed as the dialer.
537 pub handshakes_out: u64,
538 /// Handshakes this runtime accepted.
539 pub handshakes_in: u64,
540 /// Dials that reached no endpoint.
541 pub dial_failures: u64,
542 /// Sync rounds this runtime completed as the dialer.
543 pub syncs_out: u64,
544 /// Sync rounds that failed in transport.
545 pub syncs_failed: u64,
546 /// Sync rounds this runtime answered.
547 pub syncs_in: u64,
548 /// Events pushed in sync requests.
549 pub events_sent: u64,
550 /// Events served in sync answers.
551 pub events_served: u64,
552 /// Re-handshakes after an expiry or a dropped link.
553 pub reconnects: u64,
554 /// Peer restarts observed.
555 pub restarts: u64,
556 /// Links expired.
557 pub peers_expired: u64,
558 /// Claims refused for a conflict.
559 pub claims_refused: u64,
560 /// Cooperation threads alive now.
561 pub threads: u64,
562 /// Hellos and syncs this runtime refused, by code.
563 pub refused_in: BTreeMap<RefusalCode, u64>,
564 /// Refusals this runtime received or applied to a welcome, by code.
565 pub refused_out: BTreeMap<RefusalCode, u64>,
566}
567
568/// The whole cooperation runtime, as every surface reports it: the command line, the HTTP
569/// surface and MCP all render this one value, so that a person and an agent asking the same
570/// question of the same runtime are never told different things.
571///
572/// It answers in one reading the three questions a mesh raises — is this runtime
573/// cooperating at all, what does it speak, and who is it linked to — and it stays
574/// answerable when the answer is no: an inactive runtime still reports its protocol range
575/// and the reason, rather than an absence the caller has to interpret.
576///
577/// ```
578/// # use std::sync::Arc;
579/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
580/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
581/// # use majordomus_cli::mesh::identity::NodeIdentity;
582/// # use majordomus_cli::mesh::link::HttpTransport;
583/// # use majordomus_cli::mesh::registry::MeshRegistry;
584/// # use majordomus_cli::mesh::repository::of_root_commits;
585/// # let cooperation = Cooperation::new(CooperationSetup {
586/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
587/// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
588/// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
589/// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
590/// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
591/// # }).unwrap();
592/// use majordomus_cli::mesh::cooperation::CooperationStatus;
593///
594/// let running: CooperationStatus = cooperation.status();
595/// assert_eq!(running.runtime.as_deref(), Some(cooperation.runtime_key()));
596/// assert!(running.peers.is_empty(), "nothing is dialed until something dials");
597///
598/// // and where there is no runtime at all, the same shape says why
599/// let none = CooperationStatus::inactive("the mesh declaration is disabled");
600/// assert_eq!(none.protocol, running.protocol, "still answerable about what it speaks");
601/// ```
602#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
603pub struct CooperationStatus {
604 /// Whether cooperation runs in this process.
605 pub active: bool,
606 /// Why not, when it does not.
607 #[serde(default, skip_serializing_if = "Option::is_none")]
608 pub reason: Option<String>,
609 /// This runtime, `<node>-<runtime>`.
610 #[serde(default, skip_serializing_if = "Option::is_none")]
611 pub runtime: Option<String>,
612 /// This runtime's current stream.
613 #[serde(default, skip_serializing_if = "Option::is_none")]
614 pub stream: Option<StreamId>,
615 /// The repository identity links are matched on.
616 #[serde(default, skip_serializing_if = "Option::is_none")]
617 pub repository: Option<MeshRepositoryIdentity>,
618 /// The link protocol range spoken.
619 pub protocol: (u32, u32),
620 /// The link features carried.
621 pub features: Vec<String>,
622 /// Seconds between heartbeats.
623 pub heartbeat_seconds: u64,
624 /// Seconds of silence before expiry.
625 pub expiry_seconds: u64,
626 /// Where this runtime answers.
627 pub endpoints: Vec<String>,
628 /// The declared seeds.
629 pub seeds: Vec<String>,
630 /// Every linked peer, by runtime key.
631 pub peers: Vec<PeerView>,
632 /// Candidates refused, by endpoint.
633 pub refused: Vec<RefusedView>,
634 /// Counters.
635 pub counters: CooperationCounters,
636 /// The journal's tallies.
637 pub journal: JournalTallies,
638 /// When cooperation started, RFC 3339.
639 #[serde(default, skip_serializing_if = "Option::is_none")]
640 pub started_at: Option<String>,
641}
642
643impl CooperationStatus {
644 /// The status of a runtime where cooperation is not running, and why.
645 ///
646 /// The protocol range and the features are still filled in, because what this
647 /// executable would speak is knowable without a runtime and a person comparing two
648 /// machines needs it precisely when one of them is not cooperating.
649 ///
650 /// ```
651 /// use majordomus_cli::mesh::cooperation::CooperationStatus;
652 ///
653 /// let status = CooperationStatus::inactive("no mesh declaration in this repository");
654 /// assert!(!status.active);
655 /// assert!(status.reason.unwrap().contains("no mesh declaration"), "never a bare false");
656 /// assert!(!status.features.is_empty(), "what it would speak is known without a runtime");
657 /// ```
658 pub fn inactive(reason: &str) -> Self {
659 CooperationStatus {
660 active: false,
661 reason: Some(reason.into()),
662 runtime: None,
663 stream: None,
664 repository: None,
665 protocol: (LINK_PROTOCOL_MIN, LINK_PROTOCOL_MAX),
666 features: FEATURES.iter().map(|f| f.to_string()).collect(),
667 heartbeat_seconds: 0,
668 expiry_seconds: 0,
669 endpoints: Vec::new(),
670 seeds: Vec::new(),
671 peers: Vec::new(),
672 refused: Vec::new(),
673 counters: CooperationCounters::default(),
674 journal: JournalTallies::default(),
675 started_at: None,
676 }
677 }
678}
679
680/// Why a cooperation operation did not happen. Every variant is a decision rather than a
681/// fault: nothing here is a transport failure or a bug, so a caller that sees one has been
682/// told something true about the mesh and should say it to its user rather than retry.
683///
684/// `Conflict` carries the claims it met instead of a message about them, because the
685/// worker being refused needs the other side's scope, session and runtime to go and talk
686/// to whoever holds it.
687///
688/// ```
689/// # use std::sync::Arc;
690/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
691/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
692/// # use majordomus_cli::mesh::identity::NodeIdentity;
693/// # use majordomus_cli::mesh::link::HttpTransport;
694/// # use majordomus_cli::mesh::registry::MeshRegistry;
695/// # use majordomus_cli::mesh::repository::of_root_commits;
696/// # let cooperation = Cooperation::new(CooperationSetup {
697/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
698/// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
699/// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
700/// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
701/// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
702/// # }).unwrap();
703/// use majordomus_cli::mesh::cooperation::CooperationError;
704/// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
705///
706/// let held = SessionInfo::named("s1", "cli");
707/// cooperation.claim(&held, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
708///
709/// let latecomer = SessionInfo::named("s2", "cli");
710/// let refused = cooperation
711/// .claim(&latecomer, vec!["apps/cli".into()], None, ClaimMode::Exclusive, None)
712/// .unwrap_err();
713/// match &refused {
714/// CooperationError::Conflict(claims) => assert_eq!(claims[0].scope, ["apps"]),
715/// other => panic!("expected a conflict, got {other}"),
716/// }
717/// assert!(refused.to_string().contains("apps"), "the message names what it met");
718/// ```
719#[derive(Debug, Clone, thiserror::Error)]
720pub enum CooperationError {
721 /// An exclusive claim meets live exclusive claims of other sessions.
722 #[error("the scope is claimed: {}", .0.iter().map(|c| format!("{} ({})", c.key, c.scope.join(", "))).collect::<Vec<_>>().join("; "))]
723 Conflict(Vec<ClaimView>),
724 /// The operation needs something this runtime wrote, and this was written elsewhere.
725 #[error("{0}")]
726 NotOwn(String),
727 /// Nothing by that name.
728 #[error("{0}")]
729 NotFound(String),
730 /// The peer lacks a feature the operation needs.
731 #[error("{0}")]
732 FeatureUnsupported(String),
733 /// Out of bounds.
734 #[error("{0}")]
735 Invalid(String),
736}
737
738/// One verification check: what was checked, whether it holds, the evidence, and — when
739/// something is wrong or limited — the impact and what to do about it.
740///
741/// The impact and the remedy are carried with the verdict rather than left to whoever
742/// renders it, so that a failing check is actionable wherever it is read — in a terminal,
743/// in a JSON answer, or by an agent that has never seen the mesh before. A check that
744/// holds carries neither, because there is nothing to do.
745///
746/// ```
747/// # use std::sync::Arc;
748/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
749/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
750/// # use majordomus_cli::mesh::identity::NodeIdentity;
751/// # use majordomus_cli::mesh::link::HttpTransport;
752/// # use majordomus_cli::mesh::registry::MeshRegistry;
753/// # use majordomus_cli::mesh::repository::of_root_commits;
754/// # let cooperation = Cooperation::new(CooperationSetup {
755/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
756/// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
757/// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
758/// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
759/// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
760/// # }).unwrap();
761/// use majordomus_cli::mesh::cooperation::MeshVerifyCheck;
762///
763/// let report = cooperation.verify();
764/// let first: &MeshVerifyCheck = &report.checks[0];
765/// assert_eq!(first.check, "cooperation");
766/// assert!(first.detail.contains(cooperation.runtime_key()), "the evidence, not a verdict");
767/// assert!(first.remediation.is_none(), "a check that holds asks for nothing");
768/// ```
769#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
770pub struct MeshVerifyCheck {
771 /// What was checked.
772 pub check: String,
773 /// Whether it holds.
774 pub ok: bool,
775 /// The evidence.
776 pub detail: String,
777 /// What the failure or limitation means for cooperation.
778 #[serde(default, skip_serializing_if = "Option::is_none")]
779 pub impact: Option<String>,
780 /// What to do.
781 #[serde(default, skip_serializing_if = "Option::is_none")]
782 pub remediation: Option<String>,
783}
784
785/// What one sync round, run now rather than remembered, said about one peer. Verification
786/// does not report the link's standing and stop there: it exchanges with the peer and says
787/// whether the exchange worked and whether both sides ended holding the same marks, which
788/// is the difference between a peer that answers and a peer that agrees.
789///
790/// `round_trip` and `converged` are optional because a peer this runtime does not dial is
791/// not asked — an absent answer is not a failed one, and reporting `false` there would
792/// make an inbound-only link look broken every time.
793///
794/// ```
795/// # use std::collections::BTreeMap;
796/// # use std::sync::{Arc, Mutex, Weak};
797/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
798/// # use majordomus_cli::mesh::cooperation::*;
799/// # use majordomus_cli::mesh::identity::NodeIdentity;
800/// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
801/// # use majordomus_cli::mesh::registry::MeshRegistry;
802/// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
803/// # use majordomus_cli::mesh::trust::TrustPolicy;
804/// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
805/// # impl LinkTransport for Net {
806/// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
807/// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
808/// # .ok_or_else(|| format!("{e}: nobody there"))?;
809/// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
810/// # let net = Arc::new(Net::default());
811/// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
812/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
813/// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
814/// # version: "doc".into(), config: CooperationConfig::default(),
815/// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
816/// # registry: Arc::new(MeshRegistry::new()),
817/// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
818/// # checkout: CheckoutFacts::default() }).unwrap();
819/// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
820/// # let (a, b) = (runtime("a:1"), runtime("b:1"));
821/// a.dial(&["b:1".into()]).unwrap();
822///
823/// let dialer: &PeerVerdict = &a.verify().peers[0];
824/// assert_eq!(dialer.round_trip, Some(true), "asked, and answered");
825///
826/// let answerer = &b.verify().peers[0];
827/// assert!(!answerer.dialed, "B does not dial A, so B asks nothing of it");
828/// assert_eq!(answerer.round_trip, None, "and reports no verdict it did not earn");
829/// ```
830#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
831pub struct PeerVerdict {
832 /// The peer's runtime key.
833 pub runtime: String,
834 /// The link's standing after the round.
835 pub state: LinkState,
836 /// Whether this runtime dials the peer (and so ran a round with it now).
837 pub dialed: bool,
838 /// Whether the round succeeded; `None` when the peer dials this runtime instead.
839 #[serde(default, skip_serializing_if = "Option::is_none")]
840 pub round_trip: Option<bool>,
841 /// The round trip, milliseconds.
842 #[serde(default, skip_serializing_if = "Option::is_none")]
843 pub rtt_ms: Option<u64>,
844 /// Whether both sides hold the same high-water mark of every stream after the round.
845 #[serde(default, skip_serializing_if = "Option::is_none")]
846 pub converged: Option<bool>,
847 /// What happened.
848 pub detail: String,
849}
850
851/// What `mesh verify` found: the local checks, every peer's round, and the state digest
852/// afterwards. The digest is what makes the report worth running on two machines — two
853/// runtimes that print the same digest have the same sessions, claims, handovers and
854/// reviews, and two that do not have something to reconcile, whatever their links say.
855///
856/// `ok` is the conjunction of everything below it, so a caller may act on the one field
857/// and a person may read the rest to find out which part of it was false.
858///
859/// ```
860/// # use std::sync::Arc;
861/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
862/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
863/// # use majordomus_cli::mesh::identity::NodeIdentity;
864/// # use majordomus_cli::mesh::link::HttpTransport;
865/// # use majordomus_cli::mesh::registry::MeshRegistry;
866/// # use majordomus_cli::mesh::repository::of_root_commits;
867/// # let cooperation = Cooperation::new(CooperationSetup {
868/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
869/// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
870/// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
871/// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
872/// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
873/// # }).unwrap();
874/// use majordomus_cli::mesh::cooperation::MeshVerifyReport;
875///
876/// let report: MeshVerifyReport = cooperation.verify();
877/// assert_eq!(report.runtime.as_deref(), Some(cooperation.runtime_key()));
878/// assert_eq!(report.digest, Some(cooperation.state().digest), "what a peer is compared to");
879/// assert_eq!(report.ok, report.checks.iter().all(|c| c.ok), "one field for every check");
880/// ```
881#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
882pub struct MeshVerifyReport {
883 /// Whether every check holds and every round succeeded.
884 pub ok: bool,
885 /// This runtime.
886 #[serde(default, skip_serializing_if = "Option::is_none")]
887 pub runtime: Option<String>,
888 /// The checks, in the order they ran.
889 pub checks: Vec<MeshVerifyCheck>,
890 /// Every linked peer's verdict.
891 pub peers: Vec<PeerVerdict>,
892 /// The state digest after the rounds.
893 #[serde(default, skip_serializing_if = "Option::is_none")]
894 pub digest: Option<String>,
895}
896
897impl MeshVerifyReport {
898 /// The report of a runtime where cooperation does not run.
899 ///
900 /// Not running is a failing verification rather than an empty one: a person who asked
901 /// whether the mesh works has been told no, and the single check carries the impact —
902 /// nothing leaves or reaches this runtime — and the steps that would turn it on.
903 ///
904 /// ```
905 /// use majordomus_cli::mesh::cooperation::MeshVerifyReport;
906 ///
907 /// let report = MeshVerifyReport::inactive("cooperation is disabled in the declaration");
908 /// assert!(!report.ok, "a mesh that does not run has not been verified");
909 /// let check = &report.checks[0];
910 /// assert!(check.impact.is_some() && check.remediation.is_some(), "what it costs, and the fix");
911 /// ```
912 pub fn inactive(reason: &str) -> Self {
913 MeshVerifyReport {
914 ok: false,
915 runtime: None,
916 checks: vec![MeshVerifyCheck {
917 check: "cooperation".into(),
918 ok: false,
919 detail: reason.into(),
920 impact: Some("no session, claim, handover or review leaves or reaches this runtime".into()),
921 remediation: Some("enable the mesh declaration (enabled: true), trust the peers' keys (trust.allow, from `majordomus mesh identity` on each machine), and restart the server: majordomus serve stop && majordomus serve ensure".into()),
922 }],
923 peers: Vec::new(),
924 digest: None,
925 }
926 }
927}
928
929/// What an operation wrote, as its answer to the caller. Every mesh operation is an event
930/// appended to this runtime's journal, and the caller is handed the three facts it cannot
931/// work out for itself: the mesh-wide key of the thing it created, the event that says so,
932/// and the Lamport stamp that places it against everyone else's events.
933///
934/// The key is what a later operation is addressed by — a claim is released by the key its
935/// acquisition returned, never by the local name it was given — and the stamp is what a
936/// caller waiting for a peer to catch up compares against.
937///
938/// ```
939/// # use std::sync::Arc;
940/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
941/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
942/// # use majordomus_cli::mesh::identity::NodeIdentity;
943/// # use majordomus_cli::mesh::link::HttpTransport;
944/// # use majordomus_cli::mesh::registry::MeshRegistry;
945/// # use majordomus_cli::mesh::repository::of_root_commits;
946/// # let cooperation = Cooperation::new(CooperationSetup {
947/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
948/// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
949/// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
950/// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
951/// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
952/// # }).unwrap();
953/// use majordomus_cli::mesh::cooperation::Written;
954/// use majordomus_cli::mesh::journal::{ClaimMode, Durability, SessionInfo};
955///
956/// let session = SessionInfo::named("s1", "cli");
957/// let written: Written = cooperation
958/// .claim(&session, vec!["docs".into()], None, ClaimMode::Advisory, None)
959/// .unwrap();
960/// assert!(written.lamport > 0, "placed against every other runtime's events");
961/// assert_eq!(written.durability, Durability::Memory, "this journal has no file");
962///
963/// // and the key it returned is how the claim is addressed from here on
964/// cooperation.release(&written.key).unwrap();
965/// ```
966#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
967pub struct Written {
968 /// `<stream>/<local id>` of what was created or changed.
969 pub key: String,
970 /// `<stream>/<seq>` of the event that says so.
971 pub event: String,
972 /// The event's Lamport stamp.
973 pub lamport: u64,
974 /// Whether the event reached this machine's disk: `synced`, `memory` (a journal with no
975 /// file, by configuration), or `not_durable` — held and replicated to peers, not on disk
976 /// here, and not acknowledged as durable. `unknown` from an executable that predates it.
977 #[serde(default)]
978 pub durability: Durability,
979}
980
981/// What a dial or a sync round did not do. The three are kept apart because they are acted
982/// on differently: an unreachable peer is retried with a backoff, a refusal is a decision
983/// that will be made the same way next time and is recorded rather than retried, and the
984/// absence of a link is this runtime's own state and not the peer's.
985///
986/// ```
987/// # use std::sync::Arc;
988/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
989/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
990/// # use majordomus_cli::mesh::identity::NodeIdentity;
991/// # use majordomus_cli::mesh::link::HttpTransport;
992/// # use majordomus_cli::mesh::registry::MeshRegistry;
993/// # use majordomus_cli::mesh::repository::of_root_commits;
994/// # let cooperation = Cooperation::new(CooperationSetup {
995/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
996/// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
997/// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
998/// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
999/// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1000/// # }).unwrap();
1001/// use majordomus_cli::mesh::cooperation::RoundError;
1002///
1003/// // nobody answers at the discard port: a peer to back off from, not a decision
1004/// let nobody = cooperation.dial(&["127.0.0.1:9".into()]).unwrap_err();
1005/// assert!(matches!(nobody, RoundError::Unreachable(_)), "{nobody}");
1006/// assert!(nobody.to_string().contains("127.0.0.1:9"), "a failure names where it was going");
1007///
1008/// // and syncing with a peer this runtime never linked to is its own state, not the peer's
1009/// let never = cooperation.sync_with("somebody-0000000000000002").unwrap_err();
1010/// assert!(matches!(never, RoundError::NoLink), "{never}");
1011/// ```
1012#[derive(Debug, Clone)]
1013pub enum RoundError {
1014 /// No endpoint answered.
1015 Unreachable(String),
1016 /// The peer, or this runtime's check of the peer, refused.
1017 Refused(LinkRefusal),
1018 /// There is no outbound link to sync.
1019 NoLink,
1020}
1021
1022impl std::fmt::Display for RoundError {
1023 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1024 match self {
1025 RoundError::Unreachable(e) => write!(f, "unreachable: {e}"),
1026 RoundError::Refused(r) => write!(f, "refused: {r}"),
1027 RoundError::NoLink => f.write_str("no link"),
1028 }
1029 }
1030}
1031
1032struct Target {
1033 id: String,
1034 key: Option<String>,
1035 endpoints: Vec<String>,
1036}
1037
1038/// A registry record this runtime may dial, under the runtime key it would be dialed by.
1039///
1040/// Its order is the order [`MAX_TARGETS_PER_NODE`] cuts a node's runtimes in: by node, then
1041/// the runtimes answering now before the ones that stopped, then by runtime. Presence comes
1042/// before the runtime because the registry keeps a stopped runtime's record for its retention
1043/// and every runtime of a machine orders the records alike: cut by runtime alone, stopped
1044/// servers that sort first fill the cap on every live runtime at once, and no live runtime of
1045/// the node dials a live sibling until the stopped records expire.
1046struct DialCandidate<'a> {
1047 record: &'a NodeRecord,
1048 key: String,
1049}
1050
1051impl crate::order::Ordered for DialCandidate<'_> {
1052 fn order_key(&self) -> crate::order::OrderKey<'_> {
1053 let rank = match self.record.presence {
1054 Presence::Present => 0,
1055 Presence::Absent => 1,
1056 };
1057 crate::order::OrderKey::grouped(
1058 self.record.node_id.as_str(),
1059 &self.record.runtime,
1060 &self.key,
1061 )
1062 .ranked(rank)
1063 }
1064}
1065
1066/// The registry records this runtime dials: every record of `repository` that names a
1067/// runtime slot and an endpoint, whose key `trusted` accepts, other than `own_key` — at most
1068/// [`MAX_TARGETS_PER_NODE`] of one node, taken in [`DialCandidate`] order, so a node's present
1069/// runtimes are dialed before its stopped ones. The same records in any order choose the
1070/// same targets.
1071fn dial_targets(
1072 records: &[NodeRecord],
1073 repository: &str,
1074 own_key: &str,
1075 trusted: impl Fn(&str) -> bool,
1076) -> Vec<Target> {
1077 let mut candidates: Vec<DialCandidate<'_>> = records
1078 .iter()
1079 .filter(|record| {
1080 !record.runtime.is_empty()
1081 && !record.endpoints.is_empty()
1082 && record.repositories.iter().any(|r| r == repository)
1083 })
1084 .map(|record| DialCandidate {
1085 key: format!("{}-{}", record.node_id, record.runtime),
1086 record,
1087 })
1088 .filter(|candidate| candidate.key != own_key && trusted(&candidate.record.public_key))
1089 .collect();
1090 crate::order::canonical(&mut candidates);
1091 // At most MAX_TARGETS_PER_NODE runtimes of one node are dialed: runtime slots are free
1092 // to invent, and a key must not turn this runtime into a dialer of arbitrary endpoints.
1093 let mut per_node: BTreeMap<&str, usize> = BTreeMap::new();
1094 let mut targets = Vec::new();
1095 for candidate in candidates {
1096 let dialed = per_node
1097 .entry(candidate.record.node_id.as_str())
1098 .or_default();
1099 if *dialed >= MAX_TARGETS_PER_NODE {
1100 continue;
1101 }
1102 *dialed += 1;
1103 targets.push(Target {
1104 id: candidate.key.clone(),
1105 key: Some(candidate.key),
1106 endpoints: candidate.record.endpoints.clone(),
1107 });
1108 }
1109 targets
1110}
1111
1112/// One runtime's whole part in the mesh, behind one handle. It holds the single journal
1113/// this process writes to, the single table of links it has, and the admission lock that
1114/// decides claims — and it is deliberately one object rather than a set of services,
1115/// because every one of those is the other's context: a claim is admitted against state
1116/// folded from events that arrived over a link, and a link is admitted against trust the
1117/// same object evaluates.
1118///
1119/// Every surface — the command line, HTTP, MCP — performs its operations through this, so
1120/// there is one place where a session is opened and one where a claim is refused, whoever
1121/// asked. Nothing here dials or beats until [`Cooperation::start`]; a runtime that is
1122/// merely built is a complete, readable, local mesh of one.
1123///
1124/// ```
1125/// # use std::sync::Arc;
1126/// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1127/// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1128/// # use majordomus_cli::mesh::identity::NodeIdentity;
1129/// # use majordomus_cli::mesh::link::HttpTransport;
1130/// # use majordomus_cli::mesh::registry::MeshRegistry;
1131/// # use majordomus_cli::mesh::repository::of_root_commits;
1132/// # let cooperation: Arc<Cooperation> = Cooperation::new(CooperationSetup {
1133/// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1134/// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1135/// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1136/// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
1137/// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1138/// # }).unwrap();
1139/// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
1140///
1141/// let session = SessionInfo::named("s1", "cli");
1142/// cooperation.claim(&session, vec!["apps".into()], Some("documenting".into()),
1143/// ClaimMode::Exclusive, None).unwrap();
1144///
1145/// // the operation, the event and the folded state are one runtime's, with nobody linked
1146/// let state = cooperation.state();
1147/// assert_eq!(state.claims[0].intent.as_deref(), Some("documenting"));
1148/// assert_eq!(state.sessions.len(), 1, "claiming opened the session it needed");
1149/// ```
1150pub struct Cooperation {
1151 identity: Arc<NodeIdentity>,
1152 card: RuntimeCard,
1153 own_key: String,
1154 repository: MeshRepositoryIdentity,
1155 config: CooperationConfig,
1156 /// The trust in force: replaced, never patched, when the declaration changes under a
1157 /// running server ([`Cooperation::retrust`]).
1158 trust: std::sync::RwLock<TrustConfig>,
1159 registry: Arc<MeshRegistry>,
1160 transport: Arc<dyn LinkTransport>,
1161 journal: Arc<Journal>,
1162 table: Mutex<Table>,
1163 counters: Counters,
1164 stop: Arc<AtomicBool>,
1165 started_at: String,
1166 board: Option<Arc<crate::peers::PeerBoard>>,
1167 checkout: CheckoutFacts,
1168 projected: Mutex<BTreeMap<String, Projected>>,
1169 admission: Mutex<()>,
1170 rounds: Mutex<BTreeMap<String, Arc<Mutex<()>>>>,
1171}
1172
1173/// The most refusals the table lists: a stream of hellos from invented keys must not grow
1174/// memory without bound. The oldest refusal goes first.
1175const MAX_REFUSED: usize = 128;
1176
1177/// The most runtimes of one node this runtime dials.
1178pub const MAX_TARGETS_PER_NODE: usize = 8;
1179
1180/// `text` cut to at most `max` bytes on a character boundary, with control characters
1181/// removed: what the journal's single-line bounds accept, so a projection never fails
1182/// validation on a long or multi-byte title and silently retries forever.
1183fn clip(text: &str, max: usize) -> String {
1184 let mut out = String::new();
1185 for c in text.chars().filter(|c| !c.is_control()) {
1186 if out.len() + c.len_utf8() > max {
1187 break;
1188 }
1189 out.push(c);
1190 }
1191 out
1192}
1193
1194fn remember_refusal(table: &mut Table, key: String, refused: Refused) {
1195 if !table.refused.contains_key(&key) && table.refused.len() >= MAX_REFUSED {
1196 if let Some(oldest) = table
1197 .refused
1198 .iter()
1199 .min_by(|a, b| a.1.at.cmp(&b.1.at))
1200 .map(|(k, _)| k.clone())
1201 {
1202 table.refused.remove(&oldest);
1203 }
1204 }
1205 table.refused.insert(key, refused);
1206}
1207
1208fn rfc3339_now() -> String {
1209 crate::peers::rfc3339(std::time::SystemTime::now())
1210}
1211
1212/// The trust verdict for `public_key` on a machine whose own key is `own_key`: the one
1213/// definition behind [`Cooperation::trust_of_key`], and behind the journal's reload, which
1214/// runs before the runtime exists and must decide exactly what the live path decides.
1215fn key_trust(
1216 own_key: &str,
1217 config: &TrustConfig,
1218 registry: &MeshRegistry,
1219 public_key: &str,
1220) -> TrustState {
1221 if public_key == own_key {
1222 return TrustState::Trusted("this machine's own key".into());
1223 }
1224 let Some(node) = node_id_of_key(public_key) else {
1225 return TrustState::Rejected("not a key".into());
1226 };
1227 trust::evaluate(
1228 &config.policy,
1229 &config.allow,
1230 public_key,
1231 registry.known_key(&node).as_deref(),
1232 registry.trust_of(&node).as_ref(),
1233 )
1234}
1235
1236impl Cooperation {
1237 /// Open the journal and build the runtime; nothing dials until [`Cooperation::start`].
1238 ///
1239 /// The declaration is validated and this runtime's card is checked here rather than at
1240 /// the first handshake, so a runtime that would be refused by every peer refuses to
1241 /// exist instead of failing one link at a time in the background where nobody reads
1242 /// it. Building it is also what fixes this runtime's stream: the key it answers to
1243 /// from now on, and the stream its events are written to.
1244 ///
1245 /// ```
1246 /// # use std::sync::Arc;
1247 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1248 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1249 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1250 /// # use majordomus_cli::mesh::link::HttpTransport;
1251 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1252 /// # use majordomus_cli::mesh::repository::of_root_commits;
1253 /// # let setup = |heartbeat: u64, expiry: u64| CooperationSetup {
1254 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1255 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1256 /// # version: "doc".into(),
1257 /// # config: CooperationConfig { heartbeat_seconds: heartbeat, expiry_seconds: expiry,
1258 /// # ..CooperationConfig::default() },
1259 /// # trust: TrustConfig::default(), journal_path: None,
1260 /// # registry: Arc::new(MeshRegistry::new()), transport: Arc::new(HttpTransport),
1261 /// # board: None, checkout: CheckoutFacts::default() };
1262 /// let cooperation = Cooperation::new(setup(5, 30)).unwrap();
1263 /// assert!(cooperation.peers().is_empty(), "built, and dialing nobody");
1264 /// assert!(cooperation.journal().own_stream().runtime_key() == cooperation.runtime_key());
1265 ///
1266 /// // a declaration that no peer could live with is refused here, not at the first link
1267 /// let Err(refused) = Cooperation::new(setup(30, 5)) else { panic!("a live runtime") };
1268 /// assert!(refused.to_string().contains("expiry"), "{refused}");
1269 /// ```
1270 pub fn new(setup: CooperationSetup) -> Result<Arc<Self>, super::MeshError> {
1271 setup.config.validate().map_err(super::MeshError::Config)?;
1272 // The journal file is believed as far as the declared trust believes its origins
1273 // now, exactly as `origin_accept` decides for an event arriving over a link: a key
1274 // withdrawn from the allowlist does not come back from this runtime's own disk.
1275 let own_pk = setup.identity.public.public_key.as_str();
1276 let reload_accept = |event: &MeshEvent| {
1277 if key_trust(own_pk, &setup.trust, &setup.registry, &event.pk).is_trusted() {
1278 Ok(())
1279 } else {
1280 Err(Rejection::Untrusted)
1281 }
1282 };
1283 let journal = Arc::new(Journal::open_trusting(
1284 Arc::clone(&setup.identity),
1285 &setup.runtime,
1286 setup.repository.id.clone(),
1287 setup.journal_path.clone(),
1288 &reload_accept,
1289 )?);
1290 let reloaded = journal.reload_report();
1291 if reloaded.rejected_total() > 0 {
1292 tracing::warn!(
1293 accepted = reloaded.accepted,
1294 refused = ?reloaded.rejected,
1295 "mesh journal reload refused events its trust no longer admits"
1296 );
1297 }
1298 let card = RuntimeCard {
1299 pk: setup.identity.public.public_key.clone(),
1300 runtime: setup.runtime.clone(),
1301 instance: setup.identity.public.instance_id.as_str().into(),
1302 repo: setup.repository.id.clone(),
1303 name: setup.identity.public.display_name.clone(),
1304 version: setup.version.chars().take(32).collect(),
1305 features: FEATURES.iter().map(|f| f.to_string()).collect(),
1306 endpoints: setup
1307 .endpoints
1308 .iter()
1309 .take(super::protocol::MAX_ENDPOINTS)
1310 .cloned()
1311 .collect(),
1312 };
1313 card.check().map_err(super::MeshError::Protocol)?;
1314 let own_key = journal.own_stream().runtime_key();
1315 Ok(Arc::new(Cooperation {
1316 identity: setup.identity,
1317 card,
1318 own_key,
1319 repository: setup.repository,
1320 config: setup.config,
1321 trust: std::sync::RwLock::new(setup.trust),
1322 registry: setup.registry,
1323 transport: setup.transport,
1324 journal,
1325 table: Mutex::new(Table::default()),
1326 counters: Counters::default(),
1327 stop: Arc::new(AtomicBool::new(false)),
1328 started_at: rfc3339_now(),
1329 board: setup.board,
1330 checkout: setup.checkout,
1331 projected: Mutex::new(BTreeMap::new()),
1332 admission: Mutex::new(()),
1333 rounds: Mutex::new(BTreeMap::new()),
1334 }))
1335 }
1336
1337 /// Project the peer board into the journal: an attached session that is new opens a
1338 /// mesh session, a changed announcement releases its previous advisory claim and
1339 /// takes the new one, a withdrawn announcement is released, and a session that left
1340 /// the board is closed. One-way and idempotent — the board stays the local truth, the
1341 /// journal carries it to every linked runtime. Nothing happens when nothing changed.
1342 ///
1343 /// This is the whole bridge between the board a person reads on one machine and the
1344 /// mesh every other machine reads: a worker announces once, to its own server, and is
1345 /// visible everywhere without knowing that anything else exists.
1346 ///
1347 /// ```
1348 /// # use std::sync::Arc;
1349 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1350 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1351 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1352 /// # use majordomus_cli::mesh::link::HttpTransport;
1353 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1354 /// # use majordomus_cli::mesh::repository::of_root_commits;
1355 /// use majordomus_cli::peers::{ClientInfo, PeerBoard, Transport};
1356 ///
1357 /// let board = Arc::new(PeerBoard::new());
1358 /// # let cooperation = Cooperation::new(CooperationSetup {
1359 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1360 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1361 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1362 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
1363 /// # transport: Arc::new(HttpTransport), board: Some(Arc::clone(&board)),
1364 /// # checkout: CheckoutFacts::default() }).unwrap();
1365 /// let worker = board.attach(Transport::Http);
1366 /// board.identify(&worker, ClientInfo { name: "claude-code".into(), version: "1".into(),
1367 /// title: None });
1368 /// board.announce(&worker, "documenting the mesh", vec!["apps/majordomus-cli".into()]);
1369 ///
1370 /// cooperation.project_board();
1371 /// let state = cooperation.state();
1372 /// assert_eq!(state.claims[0].scope, ["apps/majordomus-cli"], "the announcement crossed");
1373 /// assert_eq!(state.sessions[0].info.client, "claude-code");
1374 ///
1375 /// // idempotent: projecting an unchanged board writes nothing
1376 /// cooperation.project_board();
1377 /// assert_eq!(cooperation.state().digest, state.digest);
1378 ///
1379 /// // and a worker that leaves takes its claim with it
1380 /// board.detach(&worker);
1381 /// cooperation.project_board();
1382 /// assert!(!cooperation.state().claims[0].state.is_live());
1383 /// ```
1384 pub fn project_board(&self) {
1385 let Some(board) = &self.board else { return };
1386 let peers = board.list();
1387 let mut projected = self.projected.lock().expect("cooperation projection");
1388 let mut seen = Vec::new();
1389 for peer in peers.iter().filter(|p| p.attached) {
1390 let session = format!("board-{}", peer.id);
1391 seen.push(session.clone());
1392 // clipped before it is compared: the projection holds what the journal accepted,
1393 // so comparing the board's whole text with it would differ on every pass for an
1394 // intent past the bound, and reopen the session each time
1395 let intent = peer.claims.first().map(|c| clip(&c.intent, 512));
1396 let entry = projected.entry(session.clone()).or_default();
1397 let fresh = entry.info.session.is_empty();
1398 if fresh || entry.info.intent != intent {
1399 let info = SessionInfo {
1400 session: session.clone(),
1401 client: clip(&peer.client.name, 64),
1402 worker: peer
1403 .client
1404 .title
1405 .clone()
1406 .filter(|t| !t.is_empty())
1407 .map(|t| clip(&t, 128)),
1408 intent: intent.clone(),
1409 checkout: self.checkout.id.clone(),
1410 branch: self.checkout.git(&["symbolic-ref", "--short", "HEAD"]),
1411 head: self.checkout.git(&["rev-parse", "--verify", "HEAD"]),
1412 ..SessionInfo::default()
1413 };
1414 if self.open_session(info.clone()).is_ok() {
1415 entry.info = info;
1416 }
1417 }
1418 let mut wanted: BTreeMap<String, (Vec<String>, String)> = BTreeMap::new();
1419 for claim in &peer.claims {
1420 let scope: Vec<String> = claim
1421 .scope
1422 .iter()
1423 .map(|p| p.trim().trim_start_matches("./").to_string())
1424 .filter(|p| !p.is_empty())
1425 .collect();
1426 if !scope.is_empty() {
1427 wanted.insert(
1428 claim.name.clone().unwrap_or_default(),
1429 (scope, claim.intent.clone()),
1430 );
1431 }
1432 }
1433 let stale: Vec<String> = entry
1434 .claims
1435 .iter()
1436 .filter(|(name, (_, scope, intent))| {
1437 wanted
1438 .get(*name)
1439 .is_none_or(|(s, i)| s != scope || i != intent)
1440 })
1441 .map(|(name, _)| name.clone())
1442 .collect();
1443 for name in stale {
1444 if let Some((key, _, _)) = entry.claims.remove(&name) {
1445 let _ = self.release(&key);
1446 }
1447 }
1448 for (name, (scope, claim_intent)) in wanted {
1449 if entry.claims.contains_key(&name) {
1450 continue;
1451 }
1452 match self.claim(
1453 &entry.info,
1454 scope.clone(),
1455 Some(clip(&claim_intent, 512)).filter(|i| !i.is_empty()),
1456 ClaimMode::Advisory,
1457 None,
1458 ) {
1459 Ok(written) => {
1460 entry
1461 .claims
1462 .insert(name, (written.key, scope, claim_intent));
1463 }
1464 Err(e) => {
1465 tracing::debug!(runtime_id = %self.own_key, session_id = %session, error = %e, "a board announcement was not projected")
1466 }
1467 }
1468 }
1469 }
1470 let gone: Vec<String> = projected
1471 .keys()
1472 .filter(|s| !seen.contains(s))
1473 .cloned()
1474 .collect();
1475 for session in gone {
1476 if let Some(entry) = projected.remove(&session) {
1477 for (key, _, _) in entry.claims.values() {
1478 let _ = self.release(key);
1479 }
1480 }
1481 let _ = self.close_session(&session);
1482 }
1483 }
1484
1485 /// This runtime's key, `<node>-<runtime>`: who it is to every other runtime, for as
1486 /// long as this checkout exists on this machine. It survives a restart — which is what
1487 /// lets a peer recognise a returning runtime as the same one rather than a second —
1488 /// while the instance inside its card does not.
1489 pub fn runtime_key(&self) -> &str {
1490 &self.own_key
1491 }
1492
1493 /// The journal this runtime writes to and every peer's events land in. Handed out
1494 /// rather than wrapped because reading the mesh is reading events: a caller that wants
1495 /// marks, tallies or the raw stream takes them from here, and the operations above
1496 /// exist for the writes, which must not be done by hand.
1497 pub fn journal(&self) -> &Arc<Journal> {
1498 &self.journal
1499 }
1500
1501 fn heartbeat(&self) -> Duration {
1502 Duration::from_secs(self.config.heartbeat_seconds)
1503 }
1504
1505 fn expiry(&self) -> Duration {
1506 Duration::from_secs(self.config.expiry_seconds)
1507 }
1508
1509 /// Start the supervisor: the heartbeat, the dialers and the expiry. Never blocks.
1510 ///
1511 /// It returns before anything has linked, because linking depends on other machines
1512 /// and a server must answer while it is still finding them. A disabled declaration
1513 /// starts nothing at all and says so through [`Cooperation::status`], rather than
1514 /// starting a supervisor that would refuse every round.
1515 ///
1516 /// ```
1517 /// # use std::sync::Arc;
1518 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1519 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1520 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1521 /// # use majordomus_cli::mesh::link::HttpTransport;
1522 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1523 /// # use majordomus_cli::mesh::repository::of_root_commits;
1524 /// # let cooperation = Cooperation::new(CooperationSetup {
1525 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1526 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1527 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1528 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
1529 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1530 /// # }).unwrap();
1531 /// cooperation.start();
1532 /// assert!(!cooperation.stopped(), "the supervisor is up and the caller was not held");
1533 ///
1534 /// cooperation.stop();
1535 /// assert!(cooperation.state().digest.len() == 32, "and the journal is still readable");
1536 /// ```
1537 pub fn start(self: &Arc<Self>) {
1538 if !self.config.enabled {
1539 return;
1540 }
1541 let me = Arc::clone(self);
1542 me.counters.threads.fetch_add(1, Ordering::SeqCst);
1543 let _ = std::thread::Builder::new()
1544 .name("majordomus-mesh-cooperation".into())
1545 .spawn(move || {
1546 tracing::info!(runtime_id = %me.own_key, repository_id = %me.repository.id, "mesh cooperation started");
1547 let mut tick: u64 = 0;
1548 while !me.stop.load(Ordering::SeqCst) {
1549 me.journal.beat_own();
1550 me.project_board();
1551 me.reconcile_workers();
1552 me.expire_links();
1553 if tick % 60 == 59 {
1554 me.journal
1555 .compact(me.expiry(), PEER_RETENTION, HANDOVER_RETENTION, false);
1556 }
1557 tick += 1;
1558 me.sleep(me.heartbeat());
1559 }
1560 me.counters.threads.fetch_sub(1, Ordering::SeqCst);
1561 });
1562 }
1563
1564 /// Stop every cooperation thread at its next bounded wait and drop every link. The
1565 /// journal stays readable; this runtime's stream simply stops beating.
1566 ///
1567 /// Stopping is deliberately not a farewell. Nothing is said to the peers, because a
1568 /// runtime that crashes says nothing either and the mesh must behave the same way in
1569 /// both cases: the streams stop beating, the claims stop excluding after the expiry,
1570 /// and no peer is left believing a promise that a power cut would have broken.
1571 ///
1572 /// ```
1573 /// # use std::sync::Arc;
1574 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1575 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1576 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1577 /// # use majordomus_cli::mesh::link::HttpTransport;
1578 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1579 /// # use majordomus_cli::mesh::repository::of_root_commits;
1580 /// # let cooperation = Cooperation::new(CooperationSetup {
1581 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1582 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1583 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1584 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
1585 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1586 /// # }).unwrap();
1587 /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
1588 ///
1589 /// let session = SessionInfo::named("s1", "cli");
1590 /// cooperation.claim(&session, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
1591 /// cooperation.stop();
1592 ///
1593 /// assert!(cooperation.stopped());
1594 /// assert_eq!(cooperation.state().claims.len(), 1, "the record survives the runtime");
1595 /// ```
1596 ///
1597 /// [`begin_stop`](Self::begin_stop) is the half of this that takes no lock, for a caller
1598 /// that must not be delayed.
1599 pub fn stop(&self) {
1600 self.begin_stop();
1601 let mut table = self.table.lock().expect("cooperation table");
1602 for flag in table.workers.values() {
1603 flag.store(true, Ordering::SeqCst);
1604 }
1605 table.workers.clear();
1606 table.inbound_links.clear();
1607 for peer in table.peers.values_mut() {
1608 peer.outbound = None;
1609 peer.inbound = None;
1610 }
1611 }
1612
1613 /// Tell every part of this runtime to stop, and return at once.
1614 ///
1615 /// [`stop`](Self::stop) also empties the link table, and to do that it must take the
1616 /// table's lock — which a worker can be holding across a dial that waits out the link
1617 /// timeout. A server shutting down must not wait for that: it has a lease to release,
1618 /// and a lease released late reads to the next process as a server that is still there.
1619 /// So the flag, which every loop and every handler checks, is set without any lock, and
1620 /// the draining is left to `stop`.
1621 ///
1622 /// ```
1623 /// # use std::sync::Arc;
1624 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1625 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1626 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1627 /// # use majordomus_cli::mesh::link::HttpTransport;
1628 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1629 /// # use majordomus_cli::mesh::repository::of_root_commits;
1630 /// # let cooperation = Cooperation::new(CooperationSetup {
1631 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1632 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1633 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1634 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
1635 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1636 /// # }).unwrap();
1637 /// cooperation.begin_stop();
1638 /// assert!(cooperation.stopped(), "every loop and handler sees it immediately");
1639 /// ```
1640 pub fn begin_stop(&self) {
1641 self.stop.store(true, Ordering::SeqCst);
1642 }
1643
1644 /// Whether this runtime has been told to stop. It is checked at the top of every
1645 /// handler, so a stopped runtime refuses a hello and a sync with a typed refusal
1646 /// instead of quietly half-linking while its threads wind down.
1647 ///
1648 /// ```
1649 /// # use std::sync::Arc;
1650 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1651 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1652 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1653 /// # use majordomus_cli::mesh::link::HttpTransport;
1654 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1655 /// # use majordomus_cli::mesh::repository::of_root_commits;
1656 /// # let cooperation = Cooperation::new(CooperationSetup {
1657 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1658 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1659 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1660 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
1661 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1662 /// # }).unwrap();
1663 /// use majordomus_cli::mesh::link::{RefusalCode, Signed};
1664 ///
1665 /// assert!(!cooperation.stopped());
1666 /// cooperation.stop();
1667 ///
1668 /// let hello = Signed { body: serde_json::json!({}), sig: String::new() };
1669 /// let reply = cooperation.accept_hello(&hello);
1670 /// assert_eq!(reply.refusal.unwrap().code, RefusalCode::NotActive, "refused, not ignored");
1671 /// ```
1672 pub fn stopped(&self) -> bool {
1673 self.stop.load(Ordering::SeqCst)
1674 }
1675
1676 fn sleep(&self, total: Duration) {
1677 let mut slept = Duration::ZERO;
1678 while slept < total && !self.stop.load(Ordering::SeqCst) {
1679 let slice = SLICE.min(total - slept);
1680 std::thread::sleep(slice);
1681 slept += slice;
1682 }
1683 }
1684
1685 // ------------------------------------------------------------------ trust
1686
1687 /// The trust verdict for a key: this machine's own key is trusted as itself; any
1688 /// other by the declared policy against what discovery knows of the node.
1689 ///
1690 /// It is asked twice for every event, once when a link is admitted and again when an
1691 /// event is ingested, because a link admitted an hour ago is not a warrant for what
1692 /// arrives over it now — an event carries the key that signed it, and a key that has
1693 /// since been withdrawn from the allowlist stops being believed without dropping
1694 /// anything.
1695 ///
1696 /// ```
1697 /// # use std::sync::Arc;
1698 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1699 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
1700 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1701 /// # use majordomus_cli::mesh::link::HttpTransport;
1702 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1703 /// # use majordomus_cli::mesh::repository::of_root_commits;
1704 /// # let identity = Arc::new(NodeIdentity::ephemeral().unwrap());
1705 /// # let own_key = identity.public.public_key.clone();
1706 /// # let cooperation = Cooperation::new(CooperationSetup {
1707 /// # identity, runtime: "0000000000000001".into(),
1708 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
1709 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
1710 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
1711 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
1712 /// # }).unwrap();
1713 /// let stranger = NodeIdentity::ephemeral().unwrap().public.public_key.clone();
1714 ///
1715 /// // two runtimes of one machine share a key, and neither has to allow the other
1716 /// assert!(cooperation.trust_of_key(&own_key).is_trusted(), "this machine, twice over");
1717 ///
1718 /// // and under the default policy an unheard-of key is not trusted by being reachable
1719 /// assert!(!cooperation.trust_of_key(&stranger).is_trusted());
1720 /// assert!(!cooperation.trust_of_key("not a key at all").is_trusted());
1721 /// ```
1722 pub fn trust_of_key(&self, public_key: &str) -> TrustState {
1723 key_trust(
1724 &self.identity.public.public_key,
1725 &self
1726 .trust
1727 .read()
1728 .unwrap_or_else(std::sync::PoisonError::into_inner),
1729 &self.registry,
1730 public_key,
1731 )
1732 }
1733
1734 /// Put `trust` in force while this runtime runs, and end every link to a runtime it no
1735 /// longer trusts. Returns the runtime keys whose links ended.
1736 ///
1737 /// Before this, a key withdrawn from the declaration kept its links until every runtime
1738 /// that had admitted it restarted. Now the server calls this whenever it reads a new
1739 /// generation of the repository: the withdrawn peer's link is gone at once, its next
1740 /// sync meets `unknown_link`, and the hello it says next is refused `untrusted`. Events it
1741 /// wrote stay in the journal until the next restart, whose reload applies the trust in
1742 /// force (I2124).
1743 ///
1744 /// ```
1745 /// use majordomus_cli::mesh::config::TrustConfig;
1746 /// # use std::sync::Arc;
1747 /// # use majordomus_cli::mesh::cooperation::{Cooperation, CooperationSetup, CheckoutFacts};
1748 /// # use majordomus_cli::mesh::{identity::NodeIdentity, registry::MeshRegistry, repository::of_root_commits};
1749 /// # use majordomus_cli::mesh::config::CooperationConfig;
1750 /// # use majordomus_cli::mesh::trust::TrustPolicy;
1751 /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed};
1752 /// # struct Nowhere;
1753 /// # impl LinkTransport for Nowhere {
1754 /// # fn post(&self, e: &str, _: &str, _: &Signed) -> Result<LinkReply, String> { Err(format!("{e}: nobody")) } }
1755 /// # let c = Cooperation::new(CooperationSetup {
1756 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
1757 /// # repository: of_root_commits(&["root".into()]), endpoints: vec![], version: "doc".into(),
1758 /// # config: CooperationConfig::default(),
1759 /// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
1760 /// # registry: Arc::new(MeshRegistry::new()), transport: Arc::new(Nowhere), board: None,
1761 /// # checkout: CheckoutFacts::default() }).unwrap();
1762 /// let stranger = NodeIdentity::ephemeral().unwrap().public.public_key;
1763 /// assert!(c.trust_of_key(&stranger).is_trusted(), "tofu");
1764 /// assert!(c.retrust(TrustConfig::default()).is_empty(), "no link to end");
1765 /// assert!(!c.trust_of_key(&stranger).is_trusted(), "deny_unknown is in force now");
1766 /// ```
1767 pub fn retrust(&self, trust: TrustConfig) -> Vec<String> {
1768 *self
1769 .trust
1770 .write()
1771 .unwrap_or_else(std::sync::PoisonError::into_inner) = trust;
1772 let mut table = self.table.lock().expect("cooperation table");
1773 let mut ended = Vec::new();
1774 let mut links = Vec::new();
1775 for (key, peer) in table.peers.iter_mut() {
1776 let verdict = self.trust_of_key(&peer.card.pk);
1777 if verdict.is_trusted() {
1778 continue;
1779 }
1780 if let Some(inbound) = peer.inbound.take() {
1781 links.push(inbound.link);
1782 }
1783 peer.outbound = None;
1784 peer.expired = true;
1785 peer.trust = verdict;
1786 ended.push(key.clone());
1787 tracing::warn!(runtime_id = %self.own_key, peer = %key, "mesh link ended: the peer's key is no longer trusted");
1788 }
1789 for link in links {
1790 table.inbound_links.remove(&link);
1791 }
1792 ended
1793 }
1794
1795 fn origin_accept(&self, event: &MeshEvent) -> Result<(), Rejection> {
1796 if self.trust_of_key(&event.pk).is_trusted() {
1797 Ok(())
1798 } else {
1799 Err(Rejection::Untrusted)
1800 }
1801 }
1802
1803 fn count_refusal(map: &Mutex<BTreeMap<RefusalCode, u64>>, code: RefusalCode) {
1804 *map.lock()
1805 .expect("refusal counters")
1806 .entry(code)
1807 .or_default() += 1;
1808 }
1809
1810 fn refuse_in(&self, code: RefusalCode, detail: String, runtime: Option<String>) -> LinkReply {
1811 Self::count_refusal(&self.counters.refused_in, code);
1812 tracing::warn!(runtime_id = %self.own_key, peer = runtime.as_deref().unwrap_or("?"), code = code.as_str(), detail = %detail, "mesh link refused");
1813 let reply = LinkReply::refused(code, detail);
1814 if let (Some(refusal), Some(runtime)) = (reply.refusal.clone(), runtime) {
1815 let mut table = self.table.lock().expect("cooperation table");
1816 remember_refusal(
1817 &mut table,
1818 format!("inbound:{runtime}"),
1819 Refused {
1820 refusal,
1821 runtime: Some(runtime),
1822 direction: "inbound",
1823 at: rfc3339_now(),
1824 },
1825 );
1826 }
1827 reply
1828 }
1829
1830 // ------------------------------------------------------------------ answering
1831
1832 /// Answer a hello: the whole admission, in order — shape, signature, freshness,
1833 /// replay, protocol, self, repository, trust, capacity — and a signed welcome.
1834 ///
1835 /// The order is the point. Everything cheap and unauthenticated is decided before
1836 /// anything expensive, and nothing a stranger says is believed before its signature is
1837 /// checked, so an unsigned flood costs a parse and a refusal. It answers a refusal
1838 /// rather than dropping the message, because a peer that is being refused for its
1839 /// clock, its repository or its key can only fix that if it is told.
1840 ///
1841 /// ```
1842 /// # use std::collections::BTreeMap;
1843 /// # use std::sync::{Arc, Mutex, Weak};
1844 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
1845 /// # use majordomus_cli::mesh::cooperation::*;
1846 /// # use majordomus_cli::mesh::identity::NodeIdentity;
1847 /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, RefusalCode, Signed, HELLO_PATH};
1848 /// # use majordomus_cli::mesh::registry::MeshRegistry;
1849 /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
1850 /// # use majordomus_cli::mesh::trust::TrustPolicy;
1851 /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
1852 /// # impl LinkTransport for Net {
1853 /// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
1854 /// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
1855 /// # .ok_or_else(|| format!("{e}: nobody there"))?;
1856 /// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
1857 /// # let net = Arc::new(Net::default());
1858 /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
1859 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
1860 /// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
1861 /// # version: "doc".into(), config: CooperationConfig::default(),
1862 /// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
1863 /// # registry: Arc::new(MeshRegistry::new()),
1864 /// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
1865 /// # checkout: CheckoutFacts::default() }).unwrap();
1866 /// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
1867 /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
1868 /// // A dials; the transport hands B's accept_hello the signed hello, and B welcomes it
1869 /// a.dial(&["b:1".into()]).unwrap();
1870 /// assert_eq!(b.peers()[0].runtime, a.runtime_key(), "admitted, both ways");
1871 ///
1872 /// // anything that is not a signed hello of this protocol is refused, and named
1873 /// let nonsense = Signed { body: serde_json::json!({"hello": true}), sig: "00".into() };
1874 /// let refusal = b.accept_hello(&nonsense).refusal.unwrap();
1875 /// assert_eq!(refusal.code, RefusalCode::Malformed);
1876 /// ```
1877 pub fn accept_hello(&self, message: &Signed) -> LinkReply {
1878 if self.stopped() {
1879 return LinkReply::refused(RefusalCode::NotActive, "cooperation is stopped");
1880 }
1881 if serde_json::to_vec(message)
1882 .map(|b| b.len())
1883 .unwrap_or(usize::MAX)
1884 > MAX_LINK_MESSAGE
1885 {
1886 return self.refuse_in(
1887 RefusalCode::Oversized,
1888 "the hello exceeds the message bound".into(),
1889 None,
1890 );
1891 }
1892 let hello: Hello = match serde_json::from_value(message.body.clone()) {
1893 Ok(h) => h,
1894 Err(e) => {
1895 return self.refuse_in(RefusalCode::Malformed, format!("not a hello: {e}"), None)
1896 }
1897 };
1898 if let Err(e) = hello.card.check() {
1899 return self.refuse_in(RefusalCode::Malformed, e, None);
1900 }
1901 let runtime = hello.card.runtime_key();
1902 if !verify_signed(&hello.card.pk, Domain::Hello, message) {
1903 return self.refuse_in(
1904 RefusalCode::Signature,
1905 "the hello's signature does not verify".into(),
1906 runtime,
1907 );
1908 }
1909 if hello.ts.abs_diff(super::protocol::now()) > MAX_LINK_SKEW_SECONDS {
1910 return self.refuse_in(
1911 RefusalCode::Stale,
1912 "the hello's clock is outside the skew window".into(),
1913 runtime,
1914 );
1915 }
1916 if hello.nonce.len() != 32 {
1917 return self.refuse_in(
1918 RefusalCode::Malformed,
1919 "the nonce is not 32 hex characters".into(),
1920 runtime,
1921 );
1922 }
1923 let Some(proto) = negotiate(hello.proto_min, hello.proto_max) else {
1924 return self.refuse_in(
1925 RefusalCode::ProtocolUnsupported,
1926 format!(
1927 "this runtime speaks link protocol {LINK_PROTOCOL_MIN}..{LINK_PROTOCOL_MAX}; the peer speaks {}..{}",
1928 hello.proto_min, hello.proto_max
1929 ),
1930 runtime,
1931 );
1932 };
1933 let runtime_key = runtime.clone().unwrap_or_default();
1934 if runtime_key == self.own_key {
1935 return self.refuse_in(
1936 RefusalCode::SelfLink,
1937 "that is this runtime".into(),
1938 runtime,
1939 );
1940 }
1941 // An addressed hello is this runtime's only when it names this runtime. The address is
1942 // inside the signed bytes, so a copy taken on its way to another runtime of the same
1943 // repository cannot be replayed here; refused before the replay cache sees its nonce.
1944 if let Some(to) = hello.to.as_deref() {
1945 if to != self.own_key {
1946 return self.refuse_in(
1947 RefusalCode::Misaddressed,
1948 "the hello is addressed to another runtime".into(),
1949 runtime,
1950 );
1951 }
1952 }
1953 if hello.card.repo != self.repository.id {
1954 return self.refuse_in(
1955 RefusalCode::RepositoryMismatch,
1956 format!(
1957 "this runtime serves repository {}; the peer serves {}",
1958 self.repository.id, hello.card.repo
1959 ),
1960 runtime,
1961 );
1962 }
1963 let verdict = self.trust_of_key(&hello.card.pk);
1964 if !verdict.is_trusted() {
1965 return self.refuse_in(
1966 RefusalCode::Untrusted,
1967 format!("the peer's key is not trusted here ({verdict:?}): discovery is not trust"),
1968 runtime,
1969 );
1970 }
1971 // The replay cache holds only nonces of trusted keys of this repository: a stranger's
1972 // hellos, however many, can neither fill it nor evict a real peer's nonce.
1973 {
1974 let mut table = self.table.lock().expect("cooperation table");
1975 let now = super::protocol::now();
1976 table
1977 .nonces
1978 .retain(|_, ts| now.saturating_sub(*ts) <= 2 * MAX_LINK_SKEW_SECONDS);
1979 let key = (hello.card.pk.clone(), hello.nonce.clone());
1980 if table.nonces.contains_key(&key) {
1981 drop(table);
1982 return self.refuse_in(
1983 RefusalCode::Replay,
1984 "this hello was already answered".into(),
1985 runtime,
1986 );
1987 }
1988 if table.nonces.len() >= NONCE_CACHE {
1989 if let Some(oldest) = table
1990 .nonces
1991 .iter()
1992 .min_by_key(|(_, ts)| **ts)
1993 .map(|(k, _)| k.clone())
1994 {
1995 table.nonces.remove(&oldest);
1996 }
1997 }
1998 table.nonces.insert(key, now);
1999 }
2000 let link = fresh_token();
2001 {
2002 let mut table = self.table.lock().expect("cooperation table");
2003 if !table.peers.contains_key(&runtime_key) && table.peers.len() >= MAX_PEERS {
2004 drop(table);
2005 return self.refuse_in(
2006 RefusalCode::Capacity,
2007 "the peer table is full".into(),
2008 runtime,
2009 );
2010 }
2011 let restarts = &self.counters.restarts;
2012 let reconnects = &self.counters.reconnects;
2013 let peer = table
2014 .peers
2015 .entry(runtime_key.clone())
2016 .or_insert_with(|| PeerLink {
2017 card: hello.card.clone(),
2018 trust: verdict.clone(),
2019 proto,
2020 outbound: None,
2021 inbound: None,
2022 last_exchange: None,
2023 linked_at: rfc3339_now(),
2024 rtt: None,
2025 failures: 0,
2026 last_error: None,
2027 handshakes: 0,
2028 reconnects: 0,
2029 restarts: 0,
2030 expired: false,
2031 });
2032 if peer.card.instance != hello.card.instance {
2033 // A new run of the same runtime: its old links are dead on both sides.
2034 peer.restarts += 1;
2035 restarts.fetch_add(1, Ordering::Relaxed);
2036 peer.outbound = None;
2037 peer.linked_at = rfc3339_now();
2038 } else if peer.expired || (peer.handshakes > 0 && peer.inbound.is_some()) {
2039 peer.reconnects += 1;
2040 reconnects.fetch_add(1, Ordering::Relaxed);
2041 }
2042 if let Some(old) = peer.inbound.take() {
2043 table.inbound_links.remove(&old.link);
2044 }
2045 let peer = table.peers.get_mut(&runtime_key).expect("inserted above");
2046 peer.card = hello.card.clone();
2047 peer.trust = verdict;
2048 peer.proto = proto;
2049 peer.inbound = Some(Inbound {
2050 link: link.clone(),
2051 counter: 0,
2052 });
2053 peer.last_exchange = Some(Instant::now());
2054 peer.handshakes += 1;
2055 peer.expired = false;
2056 table
2057 .inbound_links
2058 .insert(link.clone(), runtime_key.clone());
2059 table.refused.remove(&format!("inbound:{runtime_key}"));
2060 }
2061 self.counters.handshakes_in.fetch_add(1, Ordering::Relaxed);
2062 tracing::info!(runtime_id = %self.own_key, peer = %runtime_key, link_id = %link, "mesh link accepted");
2063 let welcome = Welcome {
2064 proto,
2065 link,
2066 card: self.card.clone(),
2067 nonce: hello.nonce,
2068 ts: super::protocol::now(),
2069 marks: self.journal.marks(),
2070 heartbeat_seconds: self.config.heartbeat_seconds,
2071 expiry_seconds: self.config.expiry_seconds,
2072 };
2073 match serde_json::to_value(&welcome) {
2074 Ok(body) => LinkReply::accepted(sign(&self.identity, Domain::Welcome, body)),
2075 Err(e) => LinkReply::refused(RefusalCode::Malformed, e.to_string()),
2076 }
2077 }
2078
2079 /// Answer a sync round: the link must be known and its peer's instance current, the
2080 /// signature must be the peer's and the counter must rise; then ingest, merge, and
2081 /// answer with what the peer lacks.
2082 ///
2083 /// The rising counter is what makes a captured round worthless: a recorded request
2084 /// replayed later carries a counter that has already been used, and is refused without
2085 /// the events in it being looked at. Answering with what the peer lacks in the same
2086 /// round is what makes the exchange symmetrical — one call, both directions — so
2087 /// convergence does not depend on which side dials.
2088 ///
2089 /// ```
2090 /// # use std::collections::BTreeMap;
2091 /// # use std::sync::{Arc, Mutex, Weak};
2092 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2093 /// # use majordomus_cli::mesh::cooperation::*;
2094 /// # use majordomus_cli::mesh::identity::NodeIdentity;
2095 /// # use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
2096 /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, RefusalCode, Signed, HELLO_PATH};
2097 /// # use majordomus_cli::mesh::registry::MeshRegistry;
2098 /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2099 /// # use majordomus_cli::mesh::trust::TrustPolicy;
2100 /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2101 /// # impl LinkTransport for Net {
2102 /// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2103 /// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2104 /// # .ok_or_else(|| format!("{e}: nobody there"))?;
2105 /// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2106 /// # let net = Arc::new(Net::default());
2107 /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2108 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2109 /// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2110 /// # version: "doc".into(), config: CooperationConfig::default(),
2111 /// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2112 /// # registry: Arc::new(MeshRegistry::new()),
2113 /// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2114 /// # checkout: CheckoutFacts::default() }).unwrap();
2115 /// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2116 /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
2117 /// let key = a.dial(&["b:1".into()]).unwrap();
2118 /// b.claim(&SessionInfo::named("s1", "cli"), vec!["docs".into()], None,
2119 /// ClaimMode::Advisory, None).unwrap();
2120 ///
2121 /// // A asks; B's answer carries B's own events back, so one round settles both sides
2122 /// a.sync_with(&key).unwrap();
2123 /// assert_eq!(a.state().claims.len(), 1, "B answered with what A lacked");
2124 /// assert_eq!(b.status().counters.syncs_in, 1);
2125 ///
2126 /// // a round under no link this runtime issued is refused before its events are read
2127 /// let stranger = Signed { body: serde_json::json!({"link": "ff", "counter": 1}), sig: "".into() };
2128 /// assert_eq!(b.accept_sync(&stranger).refusal.unwrap().code, RefusalCode::Malformed);
2129 /// ```
2130 pub fn accept_sync(&self, message: &Signed) -> LinkReply {
2131 if self.stopped() {
2132 return LinkReply::refused(RefusalCode::NotActive, "cooperation is stopped");
2133 }
2134 if serde_json::to_vec(message)
2135 .map(|b| b.len())
2136 .unwrap_or(usize::MAX)
2137 > MAX_LINK_MESSAGE
2138 {
2139 return self.refuse_in(
2140 RefusalCode::Oversized,
2141 "the sync exceeds the message bound".into(),
2142 None,
2143 );
2144 }
2145 let request: SyncRequest = match serde_json::from_value(message.body.clone()) {
2146 Ok(r) => r,
2147 Err(e) => {
2148 return self.refuse_in(
2149 RefusalCode::Malformed,
2150 format!("not a sync request: {e}"),
2151 None,
2152 )
2153 }
2154 };
2155 let runtime_key = {
2156 let mut table = self.table.lock().expect("cooperation table");
2157 let Some(key) = table.inbound_links.get(&request.link).cloned() else {
2158 drop(table);
2159 Self::count_refusal(&self.counters.refused_in, RefusalCode::UnknownLink);
2160 return LinkReply::refused(
2161 RefusalCode::UnknownLink,
2162 "no such link here: say hello",
2163 );
2164 };
2165 let peer = table
2166 .peers
2167 .get_mut(&key)
2168 .expect("an indexed link has a peer");
2169 if peer.card.instance != request.instance {
2170 drop(table);
2171 Self::count_refusal(&self.counters.refused_in, RefusalCode::UnknownLink);
2172 return LinkReply::refused(
2173 RefusalCode::UnknownLink,
2174 "the link belongs to a previous instance: say hello",
2175 );
2176 }
2177 if !verify_signed(&peer.card.pk, Domain::SyncRequest, message) {
2178 drop(table);
2179 return self.refuse_in(
2180 RefusalCode::Signature,
2181 "the sync's signature is not the linked peer's".into(),
2182 Some(key),
2183 );
2184 }
2185 let inbound = peer.inbound.as_mut().expect("an indexed link is inbound");
2186 if request.counter <= inbound.counter {
2187 drop(table);
2188 return self.refuse_in(
2189 RefusalCode::Replay,
2190 "the sync counter did not rise".into(),
2191 Some(key),
2192 );
2193 }
2194 inbound.counter = request.counter;
2195 peer.last_exchange = Some(Instant::now());
2196 peer.expired = false;
2197 key
2198 };
2199 let report = self
2200 .journal
2201 .ingest(&request.events, &|e| self.origin_accept(e));
2202 self.journal.merge_marks(
2203 &request.marks,
2204 &|pk| self.trust_of_key(pk).is_trusted(),
2205 self.expiry(),
2206 );
2207 self.counters.syncs_in.fetch_add(1, Ordering::Relaxed);
2208 if report.accepted > 0 || report.rejected_total() > 0 {
2209 tracing::debug!(runtime_id = %self.own_key, peer = %runtime_key, accepted = report.accepted, duplicate = report.duplicate, rejected = report.rejected_total(), "mesh sync ingested");
2210 }
2211 let marks = self.journal.marks_for(&request.marks);
2212 let answer =
2213 self.sync_within_bound(Domain::SyncAnswer, &request.marks, |events| SyncAnswer {
2214 link: request.link.clone(),
2215 counter: request.counter,
2216 marks: marks.clone(),
2217 events,
2218 report: report.clone(),
2219 });
2220 match answer {
2221 Ok((signed, served)) => {
2222 self.counters
2223 .events_served
2224 .fetch_add(served as u64, Ordering::Relaxed);
2225 LinkReply::accepted(signed)
2226 }
2227 Err(e) => LinkReply::refused(RefusalCode::Oversized, e),
2228 }
2229 }
2230
2231 /// Sign a sync message carrying as many of the events `peer` lacks as fit: the bytes
2232 /// left under [`MAX_LINK_MESSAGE`] once everything else in the message is serialized,
2233 /// and never more than [`SYNC_EVENT_BUDGET`]. Answers the message and how many events it
2234 /// carries.
2235 ///
2236 /// The event budget was a constant that ignored what the marks weigh: a journal at its
2237 /// stream bounds sends marks of hundreds of kilobytes, and those with a full budget of
2238 /// events made a message the peer refuses as oversized, round after round, so a busy
2239 /// journal stopped replicating (I2139). The size is checked on the message as signed,
2240 /// where it is built, and an event that would take it over is left for the next round.
2241 fn sync_within_bound<T: serde::Serialize>(
2242 &self,
2243 domain: Domain,
2244 peer: &Marks,
2245 message: impl Fn(Vec<MeshEvent>) -> T,
2246 ) -> Result<(Signed, usize), String> {
2247 let signed = |events: Vec<MeshEvent>| -> Result<(Signed, usize), String> {
2248 let body = serde_json::to_value(message(events)).map_err(json_text)?;
2249 let signed = sign(&self.identity, domain, body);
2250 // a message that cannot be written out is no message under the bound
2251 let size = serde_json::to_vec(&signed).map_or(usize::MAX, |b| b.len());
2252 Ok((signed, size))
2253 };
2254 let (bare, overhead) = signed(Vec::new())?;
2255 if overhead > MAX_LINK_MESSAGE {
2256 return Err(format!(
2257 "a sync message without events is {overhead} bytes, over the {MAX_LINK_MESSAGE}-byte link bound"
2258 ));
2259 }
2260 let budget = (MAX_LINK_MESSAGE - overhead).min(SYNC_EVENT_BUDGET);
2261 let mut events = self.journal.missing_for(peer, budget);
2262 // compact JSON puts the events in the empty array the bare message carries, one
2263 // comma between each two: the size is known before it is serialized again
2264 let sizes: Vec<usize> = events
2265 .iter()
2266 .map(|e| serde_json::to_vec(e).map_or(MAX_LINK_MESSAGE, |b| b.len()))
2267 .collect();
2268 let mut total = overhead + sizes.iter().sum::<usize>() + sizes.len().saturating_sub(1);
2269 while total > MAX_LINK_MESSAGE && !events.is_empty() {
2270 events.pop();
2271 total -= sizes[events.len()] + usize::from(!events.is_empty());
2272 }
2273 if events.is_empty() {
2274 return Ok((bare, 0));
2275 }
2276 loop {
2277 let count = events.len();
2278 let (message, size) = signed(events.clone())?;
2279 if size <= MAX_LINK_MESSAGE {
2280 return Ok((message, count));
2281 }
2282 // the budget does not count the separators, and one event always goes: drop
2283 // from the end until the message fits; the rest follow next round
2284 events.pop();
2285 if events.is_empty() {
2286 return Ok((bare, 0));
2287 }
2288 }
2289 }
2290
2291 // ------------------------------------------------------------------ dialing
2292
2293 fn refuse_out(
2294 &self,
2295 endpoint: &str,
2296 refusal: LinkRefusal,
2297 runtime: Option<String>,
2298 ) -> RoundError {
2299 Self::count_refusal(&self.counters.refused_out, refusal.code);
2300 tracing::warn!(runtime_id = %self.own_key, endpoint = %endpoint, code = refusal.code.as_str(), detail = %refusal.detail, "mesh link not established");
2301 remember_refusal(
2302 &mut self.table.lock().expect("cooperation table"),
2303 endpoint.to_string(),
2304 Refused {
2305 refusal: refusal.clone(),
2306 runtime,
2307 direction: "outbound",
2308 at: rfc3339_now(),
2309 },
2310 );
2311 RoundError::Refused(refusal)
2312 }
2313
2314 /// Say hello at each endpoint in turn until one welcomes this runtime; verify the
2315 /// welcome as strictly as a hello is verified. Returns the peer's runtime key.
2316 ///
2317 /// A peer is dialed by its endpoints rather than by one address because a machine
2318 /// answers at several and only it knows which of them a caller can reach; the key that
2319 /// comes back is what the link is remembered by, so a peer that moves between
2320 /// addresses is still the same peer. Verifying the welcome as strictly as a hello is
2321 /// what keeps the trust decision symmetrical — dialing somebody is not a reason to
2322 /// believe what answers.
2323 ///
2324 /// ```
2325 /// # use std::collections::BTreeMap;
2326 /// # use std::sync::{Arc, Mutex, Weak};
2327 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2328 /// # use majordomus_cli::mesh::cooperation::*;
2329 /// # use majordomus_cli::mesh::identity::NodeIdentity;
2330 /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
2331 /// # use majordomus_cli::mesh::registry::MeshRegistry;
2332 /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2333 /// # use majordomus_cli::mesh::trust::TrustPolicy;
2334 /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2335 /// # impl LinkTransport for Net {
2336 /// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2337 /// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2338 /// # .ok_or_else(|| format!("{e}: nobody there"))?;
2339 /// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2340 /// # let net = Arc::new(Net::default());
2341 /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2342 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2343 /// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2344 /// # version: "doc".into(), config: CooperationConfig::default(),
2345 /// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2346 /// # registry: Arc::new(MeshRegistry::new()),
2347 /// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2348 /// # checkout: CheckoutFacts::default() }).unwrap();
2349 /// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2350 /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
2351 /// // the first address is dead, the second answers: one peer, found at the one that works
2352 /// let key = a.dial(&["nowhere:1".into(), "b:1".into()]).unwrap();
2353 /// assert_eq!(key, b.runtime_key());
2354 /// assert_eq!(a.peers()[0].endpoint.as_deref(), Some("b:1"));
2355 ///
2356 /// // dialing this runtime's own endpoint is a refusal, not a link to itself
2357 /// a.dial(&["a:1".into()]).expect_err("a runtime does not cooperate with itself");
2358 /// ```
2359 pub fn dial(&self, endpoints: &[String]) -> Result<String, RoundError> {
2360 self.dial_to(None, endpoints)
2361 }
2362
2363 /// [`Cooperation::dial`], addressed: when the dialer knows the runtime key it means to
2364 /// reach, the hello names it inside its signed bytes, and any other runtime that receives
2365 /// a copy refuses it as `misaddressed`.
2366 pub(crate) fn dial_to(
2367 &self,
2368 to: Option<&str>,
2369 endpoints: &[String],
2370 ) -> Result<String, RoundError> {
2371 let hello = Hello {
2372 proto_min: LINK_PROTOCOL_MIN,
2373 proto_max: LINK_PROTOCOL_MAX,
2374 card: self.card.clone(),
2375 nonce: fresh_token(),
2376 ts: super::protocol::now(),
2377 to: to.map(str::to_string),
2378 };
2379 let body =
2380 serde_json::to_value(&hello).map_err(|e| RoundError::Unreachable(e.to_string()))?;
2381 let message = sign(&self.identity, Domain::Hello, body);
2382 let mut last = "no endpoint".to_string();
2383 for endpoint in endpoints {
2384 let started = Instant::now();
2385 let reply = match self.transport.post(endpoint, HELLO_PATH, &message) {
2386 Ok(reply) => reply,
2387 Err(e) => {
2388 last = e;
2389 continue;
2390 }
2391 };
2392 if let Some(refusal) = reply.refusal {
2393 return Err(self.refuse_out(endpoint, refusal, None));
2394 }
2395 let Some(signed) = reply.signed else {
2396 return Err(self.refuse_out(
2397 endpoint,
2398 LinkRefusal::new(RefusalCode::Malformed, "an empty reply"),
2399 None,
2400 ));
2401 };
2402 let welcome: Welcome = match serde_json::from_value(signed.body.clone()) {
2403 Ok(w) => w,
2404 Err(e) => {
2405 return Err(self.refuse_out(
2406 endpoint,
2407 LinkRefusal::new(RefusalCode::Malformed, format!("not a welcome: {e}")),
2408 None,
2409 ))
2410 }
2411 };
2412 if let Err(e) = welcome.card.check() {
2413 return Err(self.refuse_out(
2414 endpoint,
2415 LinkRefusal::new(RefusalCode::Malformed, e),
2416 None,
2417 ));
2418 }
2419 let runtime = welcome.card.runtime_key();
2420 let check = if !verify_signed(&welcome.card.pk, Domain::Welcome, &signed) {
2421 Some((
2422 RefusalCode::Signature,
2423 "the welcome's signature does not verify".to_string(),
2424 ))
2425 } else if welcome.nonce != hello.nonce {
2426 Some((
2427 RefusalCode::Replay,
2428 "the welcome answers another hello".into(),
2429 ))
2430 } else if welcome.ts.abs_diff(super::protocol::now()) > MAX_LINK_SKEW_SECONDS {
2431 Some((
2432 RefusalCode::Stale,
2433 "the welcome's clock is outside the skew window".into(),
2434 ))
2435 } else if negotiate(welcome.proto, welcome.proto).is_none() {
2436 Some((
2437 RefusalCode::ProtocolUnsupported,
2438 format!("the peer chose protocol {}", welcome.proto),
2439 ))
2440 } else if runtime.as_deref() == Some(self.own_key.as_str()) {
2441 Some((
2442 RefusalCode::SelfLink,
2443 "that endpoint is this runtime".into(),
2444 ))
2445 } else if welcome.card.repo != self.repository.id {
2446 Some((
2447 RefusalCode::RepositoryMismatch,
2448 format!("the peer serves repository {}", welcome.card.repo),
2449 ))
2450 } else if !self.trust_of_key(&welcome.card.pk).is_trusted() {
2451 Some((
2452 RefusalCode::Untrusted,
2453 "the peer's key is not trusted here: discovery is not trust".into(),
2454 ))
2455 } else {
2456 None
2457 };
2458 if let Some((code, detail)) = check {
2459 return Err(self.refuse_out(endpoint, LinkRefusal::new(code, detail), runtime));
2460 }
2461 let key = runtime.expect("a checked card has a runtime key");
2462 let verdict = self.trust_of_key(&welcome.card.pk);
2463 let mut table = self.table.lock().expect("cooperation table");
2464 if !table.peers.contains_key(&key) && table.peers.len() >= MAX_PEERS {
2465 drop(table);
2466 return Err(self.refuse_out(
2467 endpoint,
2468 LinkRefusal::new(RefusalCode::Capacity, "the peer table is full"),
2469 Some(key),
2470 ));
2471 }
2472 let peer = table.peers.entry(key.clone()).or_insert_with(|| PeerLink {
2473 card: welcome.card.clone(),
2474 trust: verdict.clone(),
2475 proto: welcome.proto,
2476 outbound: None,
2477 inbound: None,
2478 last_exchange: None,
2479 linked_at: rfc3339_now(),
2480 rtt: None,
2481 failures: 0,
2482 last_error: None,
2483 handshakes: 0,
2484 reconnects: 0,
2485 restarts: 0,
2486 expired: false,
2487 });
2488 if peer.card.instance != welcome.card.instance {
2489 peer.restarts += 1;
2490 self.counters.restarts.fetch_add(1, Ordering::Relaxed);
2491 peer.linked_at = rfc3339_now();
2492 } else if peer.handshakes > 0 {
2493 peer.reconnects += 1;
2494 self.counters.reconnects.fetch_add(1, Ordering::Relaxed);
2495 }
2496 peer.card = welcome.card.clone();
2497 peer.trust = verdict;
2498 peer.proto = welcome.proto;
2499 peer.outbound = Some(Outbound {
2500 link: welcome.link.clone(),
2501 endpoint: endpoint.clone(),
2502 counter: 0,
2503 remote_marks: welcome.marks.clone(),
2504 });
2505 peer.last_exchange = Some(Instant::now());
2506 peer.rtt = Some(started.elapsed());
2507 peer.failures = 0;
2508 peer.last_error = None;
2509 peer.handshakes += 1;
2510 peer.expired = false;
2511 table.refused.remove(endpoint);
2512 drop(table);
2513 self.journal.merge_marks(
2514 &welcome.marks,
2515 &|pk| self.trust_of_key(pk).is_trusted(),
2516 self.expiry(),
2517 );
2518 self.counters.handshakes_out.fetch_add(1, Ordering::Relaxed);
2519 tracing::info!(runtime_id = %self.own_key, peer = %key, endpoint = %endpoint, link_id = %welcome.link, "mesh link established");
2520 return Ok(key);
2521 }
2522 self.counters.dial_failures.fetch_add(1, Ordering::Relaxed);
2523 Err(RoundError::Unreachable(last))
2524 }
2525
2526 /// One sync round with a peer this runtime dials: push what the peer lacks, take what
2527 /// this runtime lacks, in one exchange.
2528 ///
2529 /// Only the dialing side runs rounds, so a link has one driver and one counter however
2530 /// many threads hold it; an inbound-only peer is synced by its own side. A round is
2531 /// bounded by an event budget rather than by time, so a runtime that has been away for
2532 /// a day catches up over several rounds instead of one that never finishes.
2533 ///
2534 /// ```
2535 /// # use std::collections::BTreeMap;
2536 /// # use std::sync::{Arc, Mutex, Weak};
2537 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2538 /// # use majordomus_cli::mesh::cooperation::*;
2539 /// # use majordomus_cli::mesh::identity::NodeIdentity;
2540 /// # use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
2541 /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
2542 /// # use majordomus_cli::mesh::registry::MeshRegistry;
2543 /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2544 /// # use majordomus_cli::mesh::trust::TrustPolicy;
2545 /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2546 /// # impl LinkTransport for Net {
2547 /// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2548 /// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2549 /// # .ok_or_else(|| format!("{e}: nobody there"))?;
2550 /// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2551 /// # let net = Arc::new(Net::default());
2552 /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2553 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2554 /// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2555 /// # version: "doc".into(), config: CooperationConfig::default(),
2556 /// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2557 /// # registry: Arc::new(MeshRegistry::new()),
2558 /// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2559 /// # checkout: CheckoutFacts::default() }).unwrap();
2560 /// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2561 /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
2562 /// let key = a.dial(&["b:1".into()]).unwrap();
2563 /// a.claim(&SessionInfo::named("s1", "cli"), vec!["apps".into()], None,
2564 /// ClaimMode::Exclusive, None).unwrap();
2565 ///
2566 /// for _ in 0..2 {
2567 /// a.journal().beat_own();
2568 /// a.sync_with(&key).unwrap();
2569 /// }
2570 /// assert_eq!(a.state().digest, b.state().digest, "the two now hold one state");
2571 ///
2572 /// // B's own claims come back over the same rounds, without B dialing anybody
2573 /// b.claim(&SessionInfo::named("s2", "cli"), vec!["docs".into()], None,
2574 /// ClaimMode::Exclusive, None).unwrap();
2575 /// b.journal().beat_own();
2576 /// a.sync_with(&key).unwrap();
2577 /// assert_eq!(a.state().claims.len(), 2);
2578 /// ```
2579 pub fn sync_with(&self, runtime_key: &str) -> Result<(), RoundError> {
2580 // One round per link at a time: a verification and a worker (or two workers for one
2581 // peer) racing on the counter would refuse each other's rounds as replays.
2582 let round = {
2583 let mut rounds = self.rounds.lock().expect("sync rounds");
2584 Arc::clone(rounds.entry(runtime_key.to_string()).or_default())
2585 };
2586 let _round = round.lock().expect("sync round");
2587 let (out, pk) = {
2588 let mut table = self.table.lock().expect("cooperation table");
2589 let Some(peer) = table.peers.get_mut(runtime_key) else {
2590 return Err(RoundError::NoLink);
2591 };
2592 let Some(out) = peer.outbound.as_mut() else {
2593 return Err(RoundError::NoLink);
2594 };
2595 out.counter += 1;
2596 (out.clone(), peer.card.pk.clone())
2597 };
2598 let marks = self.journal.marks_for(&out.remote_marks);
2599 let (message, sent) = self
2600 .sync_within_bound(Domain::SyncRequest, &out.remote_marks, |events| {
2601 SyncRequest {
2602 link: out.link.clone(),
2603 counter: out.counter,
2604 instance: self.card.instance.clone(),
2605 marks: marks.clone(),
2606 events,
2607 }
2608 })
2609 .map_err(RoundError::Unreachable)?;
2610 let sent = sent as u64;
2611 // The push is counted as the request leaves, not when its answer returns. The peer
2612 // holds these events from the moment it ingests them, before its answer is written,
2613 // so a count taken on the answer trails every reading of the peer: in between, the
2614 // events are held there and not yet sent here, and the count that lands a moment
2615 // later reads as an event that travelled twice. The peer counts what it serves the
2616 // same way, as it forms the answer. A round that fails after this has still put its
2617 // events on the wire, and a round that sends them again counts them again.
2618 self.counters.events_sent.fetch_add(sent, Ordering::Relaxed);
2619 let started = Instant::now();
2620 let failed = |why: String| {
2621 self.counters.syncs_failed.fetch_add(1, Ordering::Relaxed);
2622 let mut table = self.table.lock().expect("cooperation table");
2623 if let Some(peer) = table.peers.get_mut(runtime_key) {
2624 peer.failures += 1;
2625 peer.last_error = Some(why.clone());
2626 }
2627 why
2628 };
2629 let reply = match self.transport.post(&out.endpoint, SYNC_PATH, &message) {
2630 Ok(reply) => reply,
2631 Err(e) => return Err(RoundError::Unreachable(failed(e))),
2632 };
2633 if let Some(refusal) = reply.refusal {
2634 failed(refusal.to_string());
2635 if refusal.code == RefusalCode::UnknownLink {
2636 let mut table = self.table.lock().expect("cooperation table");
2637 if let Some(peer) = table.peers.get_mut(runtime_key) {
2638 peer.outbound = None;
2639 }
2640 }
2641 Self::count_refusal(&self.counters.refused_out, refusal.code);
2642 return Err(RoundError::Refused(refusal));
2643 }
2644 let Some(signed) = reply.signed else {
2645 return Err(RoundError::Unreachable(failed(
2646 "an empty sync reply".into(),
2647 )));
2648 };
2649 let answer: SyncAnswer = match serde_json::from_value(signed.body.clone()) {
2650 Ok(a) => a,
2651 Err(e) => {
2652 return Err(RoundError::Unreachable(failed(format!(
2653 "not a sync answer: {e}"
2654 ))))
2655 }
2656 };
2657 if !verify_signed(&pk, Domain::SyncAnswer, &signed)
2658 || answer.link != out.link
2659 || answer.counter != out.counter
2660 {
2661 return Err(RoundError::Refused(LinkRefusal::new(
2662 RefusalCode::Signature,
2663 failed("the sync answer is not the linked peer's answer to this round".into()),
2664 )));
2665 }
2666 self.journal
2667 .ingest(&answer.events, &|e| self.origin_accept(e));
2668 self.journal.merge_marks(
2669 &answer.marks,
2670 &|pk| self.trust_of_key(pk).is_trusted(),
2671 self.expiry(),
2672 );
2673 {
2674 let mut table = self.table.lock().expect("cooperation table");
2675 if let Some(peer) = table.peers.get_mut(runtime_key) {
2676 if let Some(o) = peer.outbound.as_mut() {
2677 o.remote_marks = answer.marks;
2678 }
2679 peer.last_exchange = Some(Instant::now());
2680 peer.rtt = Some(started.elapsed());
2681 peer.failures = 0;
2682 peer.last_error = None;
2683 peer.expired = false;
2684 }
2685 }
2686 self.counters.syncs_out.fetch_add(1, Ordering::Relaxed);
2687 Ok(())
2688 }
2689
2690 // ------------------------------------------------------------------ supervision
2691
2692 fn targets(&self) -> Vec<Target> {
2693 let mut targets: Vec<Target> = self
2694 .config
2695 .seeds
2696 .iter()
2697 .map(|seed| Target {
2698 id: format!("seed:{seed}"),
2699 key: None,
2700 endpoints: vec![seed.trim_start_matches("http://").to_string()],
2701 })
2702 .collect();
2703 targets.extend(dial_targets(
2704 &self.registry.list(),
2705 &self.repository.id,
2706 &self.own_key,
2707 |public_key| self.trust_of_key(public_key).is_trusted(),
2708 ));
2709 targets
2710 }
2711
2712 fn reconcile_workers(self: &Arc<Self>) {
2713 let targets = self.targets();
2714 let mut table = self.table.lock().expect("cooperation table");
2715 let wanted: Vec<String> = targets.iter().map(|t| t.id.clone()).collect();
2716 table.workers.retain(|id, flag| {
2717 let keep = wanted.contains(id);
2718 if !keep {
2719 flag.store(true, Ordering::SeqCst);
2720 }
2721 keep
2722 });
2723 for target in targets {
2724 if table.workers.contains_key(&target.id) {
2725 continue;
2726 }
2727 let flag = Arc::new(AtomicBool::new(false));
2728 table.workers.insert(target.id.clone(), Arc::clone(&flag));
2729 let me = Arc::clone(self);
2730 me.counters.threads.fetch_add(1, Ordering::SeqCst);
2731 let _ = std::thread::Builder::new()
2732 .name("majordomus-mesh-link".into())
2733 .spawn(move || {
2734 me.work(target, flag);
2735 me.counters.threads.fetch_sub(1, Ordering::SeqCst);
2736 });
2737 }
2738 }
2739
2740 /// Whether this runtime should dial a known peer: the lower runtime key dials, and
2741 /// the higher one dials only when nothing has arrived from the peer lately — so a
2742 /// pair keeps one link when both can reach each other, and two when only one can.
2743 fn should_dial(&self, key: &str) -> bool {
2744 if self.own_key.as_str() < key {
2745 return true;
2746 }
2747 let table = self.table.lock().expect("cooperation table");
2748 let quiet = 3 * self.heartbeat();
2749 !table.peers.get(key).is_some_and(|p| {
2750 p.inbound.is_some() && p.last_exchange.is_some_and(|t| t.elapsed() <= quiet)
2751 })
2752 }
2753
2754 fn outbound_key(&self, key: &str) -> bool {
2755 let table = self.table.lock().expect("cooperation table");
2756 table.peers.get(key).is_some_and(|p| p.outbound.is_some())
2757 }
2758
2759 fn work(&self, target: Target, flag: Arc<AtomicBool>) {
2760 let mut learned = target.key.clone();
2761 let mut failures: u32 = 0;
2762 // An answerer that restarted forgets the link and answers `unknown_link` once; the
2763 // dialer says hello again at once. One that keeps answering it is retried no faster
2764 // than a heartbeat, or a peer that welcomes and refuses every sync drives a busy loop.
2765 let mut unknown_in_a_row: u32 = 0;
2766 while !self.stopped() && !flag.load(Ordering::SeqCst) {
2767 let linked = learned
2768 .as_deref()
2769 .filter(|k| self.outbound_key(k))
2770 .map(str::to_string);
2771 let outcome = match linked {
2772 Some(key) => self.sync_with(&key),
2773 None => {
2774 let may = target.key.as_deref().is_none_or(|k| self.should_dial(k));
2775 if !may {
2776 self.sleep(self.heartbeat());
2777 continue;
2778 }
2779 // addressed whenever the runtime behind the endpoints is known
2780 let to = learned.as_deref().or(target.key.as_deref());
2781 match self.dial_to(to, &target.endpoints) {
2782 Ok(key) => {
2783 learned = Some(key.clone());
2784 // A fresh link syncs at once: the peer should see this
2785 // runtime's state now, not a heartbeat from now.
2786 self.sync_with(&key)
2787 }
2788 Err(e) => Err(e),
2789 }
2790 }
2791 };
2792 let delay = self.after_round(&outcome, &mut failures, &mut unknown_in_a_row);
2793 self.sleep(delay);
2794 }
2795 }
2796
2797 /// How long [`Self::work`] waits after a round that ended in `outcome`, with the counts of
2798 /// failed rounds and of `unknown_link` answers in a row carried from round to round.
2799 fn after_round(
2800 &self,
2801 outcome: &Result<(), RoundError>,
2802 failures: &mut u32,
2803 unknown_in_a_row: &mut u32,
2804 ) -> Duration {
2805 match outcome {
2806 Ok(()) => {
2807 *failures = 0;
2808 *unknown_in_a_row = 0;
2809 self.heartbeat()
2810 }
2811 Err(RoundError::Refused(r)) if r.code == RefusalCode::UnknownLink => {
2812 *failures = 0;
2813 *unknown_in_a_row = unknown_in_a_row.saturating_add(1);
2814 if *unknown_in_a_row == 1 {
2815 Duration::ZERO
2816 } else {
2817 self.heartbeat()
2818 }
2819 }
2820 Err(RoundError::Refused(_)) => {
2821 *failures = failures.saturating_add(3).min(6);
2822 self.backoff(*failures)
2823 }
2824 Err(_) => {
2825 *failures = failures.saturating_add(1).min(6);
2826 self.backoff(*failures)
2827 }
2828 }
2829 }
2830
2831 /// Heartbeat × 2^failures, never beyond the expiry: an unreachable peer is asked less
2832 /// and less, and still within one expiry of coming back.
2833 fn backoff(&self, failures: u32) -> Duration {
2834 let factor = 1u32 << failures.min(6);
2835 (self.heartbeat() * factor).min(self.expiry().max(self.heartbeat()))
2836 }
2837
2838 fn expire_links(&self) {
2839 let expiry = self.expiry();
2840 let mut table = self.table.lock().expect("cooperation table");
2841 let mut dropped_links = Vec::new();
2842 let mut gone = Vec::new();
2843 for (key, peer) in table.peers.iter_mut() {
2844 let age = peer.last_exchange.map(|t| t.elapsed());
2845 if age.is_some_and(|a| a > expiry) && !peer.expired {
2846 peer.expired = true;
2847 if let Some(inbound) = peer.inbound.take() {
2848 dropped_links.push(inbound.link);
2849 }
2850 peer.outbound = None;
2851 self.counters.peers_expired.fetch_add(1, Ordering::Relaxed);
2852 tracing::warn!(runtime_id = %self.own_key, peer = %key, "mesh link expired: no exchange within the expiry");
2853 }
2854 if age.is_some_and(|a| a > expiry + PEER_RETENTION) {
2855 gone.push(key.clone());
2856 }
2857 }
2858 for link in dropped_links {
2859 table.inbound_links.remove(&link);
2860 }
2861 for key in gone {
2862 table.peers.remove(&key);
2863 }
2864 }
2865
2866 fn state_of(&self, peer: &PeerLink) -> LinkState {
2867 let Some(age) = peer.last_exchange.map(|t| t.elapsed()) else {
2868 return LinkState::Connecting;
2869 };
2870 let expiry = self.expiry();
2871 if peer.expired || age > expiry {
2872 LinkState::Expired
2873 } else if age <= 2 * self.heartbeat() + Duration::from_secs(1) {
2874 LinkState::Connected
2875 } else if age <= expiry / 2 {
2876 LinkState::Degraded
2877 } else {
2878 LinkState::Unreachable
2879 }
2880 }
2881
2882 // ------------------------------------------------------------------ reading
2883
2884 /// Every peer this runtime has a link to, including the ones that have expired and not
2885 /// yet been forgotten. An expired peer is kept for a retention window rather than
2886 /// dropped on the spot, because "the peer I had is gone" is the answer a person needs
2887 /// when a machine stops answering, and a list that silently shrinks tells them nothing.
2888 ///
2889 /// ```
2890 /// # use std::collections::BTreeMap;
2891 /// # use std::sync::{Arc, Mutex, Weak};
2892 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2893 /// # use majordomus_cli::mesh::cooperation::*;
2894 /// # use majordomus_cli::mesh::identity::NodeIdentity;
2895 /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
2896 /// # use majordomus_cli::mesh::registry::MeshRegistry;
2897 /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
2898 /// # use majordomus_cli::mesh::trust::TrustPolicy;
2899 /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
2900 /// # impl LinkTransport for Net {
2901 /// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
2902 /// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
2903 /// # .ok_or_else(|| format!("{e}: nobody there"))?;
2904 /// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
2905 /// # let net = Arc::new(Net::default());
2906 /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
2907 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
2908 /// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
2909 /// # version: "doc".into(), config: CooperationConfig::default(),
2910 /// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
2911 /// # registry: Arc::new(MeshRegistry::new()),
2912 /// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
2913 /// # checkout: CheckoutFacts::default() }).unwrap();
2914 /// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
2915 /// # let (a, b, c) = (runtime("a:1"), runtime("b:1"), runtime("c:1"));
2916 /// a.dial(&["b:1".into()]).unwrap();
2917 /// a.dial(&["c:1".into()]).unwrap();
2918 ///
2919 /// let keys: Vec<String> = a.peers().into_iter().map(|p| p.runtime).collect();
2920 /// assert_eq!(keys.len(), 2, "one entry per runtime, whoever dialed whom");
2921 /// assert!(keys.iter().any(|k| k == b.runtime_key()), "B is one of them");
2922 /// assert!(keys.iter().any(|k| k == c.runtime_key()), "and C is the other");
2923 /// assert_eq!(b.peers().len(), 1, "B was dialed once and holds one peer");
2924 /// ```
2925 pub fn peers(&self) -> Vec<PeerView> {
2926 let table = self.table.lock().expect("cooperation table");
2927 let own_node = self.identity.public.node_id.as_str();
2928 table
2929 .peers
2930 .iter()
2931 .map(|(key, peer)| PeerView {
2932 runtime: key.clone(),
2933 node: key[..32].to_string(),
2934 instance: peer.card.instance.clone(),
2935 name: peer.card.name.clone(),
2936 version: peer.card.version.clone(),
2937 local: &key[..32] == own_node,
2938 trust: peer.trust.clone(),
2939 protocol: peer.proto,
2940 features: peer.card.features.clone(),
2941 endpoints: peer.card.endpoints.clone(),
2942 state: self.state_of(peer),
2943 outbound: peer.outbound.is_some(),
2944 inbound: peer.inbound.is_some(),
2945 endpoint: peer.outbound.as_ref().map(|o| o.endpoint.clone()),
2946 last_exchange_ms: peer.last_exchange.map(|t| t.elapsed().as_millis() as u64),
2947 rtt_ms: peer.rtt.map(|d| d.as_millis() as u64),
2948 failures: peer.failures,
2949 last_error: peer.last_error.clone(),
2950 handshakes: peer.handshakes,
2951 reconnects: peer.reconnects,
2952 restarts: peer.restarts,
2953 linked_at: peer.linked_at.clone(),
2954 })
2955 .collect()
2956 }
2957
2958 /// Everything a person or an agent can ask about this runtime, in one reading: what it
2959 /// is, what it speaks, who it is linked to, what it refused, and what it has done.
2960 ///
2961 /// It is one call rather than several because the parts are only meaningful together —
2962 /// a peer count means nothing without the refusals beside it — and because two calls
2963 /// would be two moments, and a mesh changes between them.
2964 ///
2965 /// ```
2966 /// # use std::sync::Arc;
2967 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
2968 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
2969 /// # use majordomus_cli::mesh::identity::NodeIdentity;
2970 /// # use majordomus_cli::mesh::link::HttpTransport;
2971 /// # use majordomus_cli::mesh::registry::MeshRegistry;
2972 /// # use majordomus_cli::mesh::repository::of_root_commits;
2973 /// # let repository = of_root_commits(&["root".into()]);
2974 /// # let cooperation = Cooperation::new(CooperationSetup {
2975 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
2976 /// # repository: repository.clone(), endpoints: vec!["127.0.0.1:9".into()],
2977 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
2978 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
2979 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
2980 /// # }).unwrap();
2981 /// let status = cooperation.status();
2982 /// assert!(status.active, "this process is cooperating");
2983 /// assert_eq!(status.repository.unwrap().id, repository.id, "what links are matched on");
2984 /// assert_eq!(status.endpoints, ["127.0.0.1:9"], "where a peer would dial it");
2985 /// assert!(status.started_at.is_some(), "and since when");
2986 /// ```
2987 pub fn status(&self) -> CooperationStatus {
2988 let refused = {
2989 let table = self.table.lock().expect("cooperation table");
2990 table
2991 .refused
2992 .iter()
2993 .map(|(endpoint, r)| RefusedView {
2994 endpoint: endpoint.clone(),
2995 direction: r.direction.into(),
2996 runtime: r.runtime.clone(),
2997 refusal: r.refusal.clone(),
2998 at: r.at.clone(),
2999 })
3000 .collect()
3001 };
3002 let c = &self.counters;
3003 let load = |a: &AtomicU64| a.load(Ordering::Relaxed);
3004 CooperationStatus {
3005 active: !self.stopped() && self.config.enabled,
3006 reason: if self.stopped() {
3007 Some("stopped".into())
3008 } else if !self.config.enabled {
3009 Some("cooperation is disabled in the mesh declaration".into())
3010 } else {
3011 None
3012 },
3013 runtime: Some(self.own_key.clone()),
3014 stream: Some(self.journal.own_stream().clone()),
3015 repository: Some(self.repository.clone()),
3016 protocol: (LINK_PROTOCOL_MIN, LINK_PROTOCOL_MAX),
3017 features: self.card.features.clone(),
3018 heartbeat_seconds: self.config.heartbeat_seconds,
3019 expiry_seconds: self.config.expiry_seconds,
3020 endpoints: self.card.endpoints.clone(),
3021 seeds: self.config.seeds.clone(),
3022 peers: self.peers(),
3023 refused,
3024 counters: CooperationCounters {
3025 handshakes_out: load(&c.handshakes_out),
3026 handshakes_in: load(&c.handshakes_in),
3027 dial_failures: load(&c.dial_failures),
3028 syncs_out: load(&c.syncs_out),
3029 syncs_failed: load(&c.syncs_failed),
3030 syncs_in: load(&c.syncs_in),
3031 events_sent: load(&c.events_sent),
3032 events_served: load(&c.events_served),
3033 reconnects: load(&c.reconnects),
3034 restarts: load(&c.restarts),
3035 peers_expired: load(&c.peers_expired),
3036 claims_refused: load(&c.claims_refused),
3037 threads: c.threads.load(Ordering::SeqCst),
3038 refused_in: c.refused_in.lock().expect("counters").clone(),
3039 refused_out: c.refused_out.lock().expect("counters").clone(),
3040 },
3041 journal: self.journal.tallies(),
3042 started_at: Some(self.started_at.clone()),
3043 }
3044 }
3045
3046 /// The folded state, liveness judged on this runtime's clock now.
3047 ///
3048 /// It is computed on every call rather than kept, because the answer changes with the
3049 /// clock and not only with the events: nothing has to happen for a claim to expire, so
3050 /// a cached state would go quietly wrong while nothing was arriving. Two runtimes that
3051 /// hold the same events and agree about who is alive fold the same state, which is what
3052 /// the digest is for.
3053 ///
3054 /// ```
3055 /// # use std::sync::Arc;
3056 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3057 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3058 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3059 /// # use majordomus_cli::mesh::link::HttpTransport;
3060 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3061 /// # use majordomus_cli::mesh::repository::of_root_commits;
3062 /// # let cooperation = Cooperation::new(CooperationSetup {
3063 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3064 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3065 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3066 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3067 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3068 /// # }).unwrap();
3069 /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3070 ///
3071 /// let empty = cooperation.state();
3072 /// assert!(empty.claims.is_empty());
3073 ///
3074 /// let session = SessionInfo::named("s1", "cli");
3075 /// let claim = cooperation
3076 /// .claim(&session, vec!["apps".into()], None, ClaimMode::Exclusive, None)
3077 /// .unwrap();
3078 /// let taken = cooperation.state();
3079 /// assert!(taken.claims[0].state.is_live());
3080 /// assert_ne!(taken.digest, empty.digest, "the digest is what a peer is compared to");
3081 ///
3082 /// cooperation.release(&claim.key).unwrap();
3083 /// assert!(!cooperation.state().claims[0].state.is_live(), "released, not forgotten");
3084 /// ```
3085 pub fn state(&self) -> CooperationState {
3086 let live = self.journal.stream_liveness(self.expiry());
3087 fold(&self.journal.events(), &|s| {
3088 live.get(s)
3089 .map(|(l, _)| *l)
3090 .unwrap_or(StreamLiveness::Expired)
3091 })
3092 }
3093
3094 /// Every stream's liveness and the milliseconds since its beat rose — the layer claims
3095 /// actually follow, which is not the layer links live on.
3096 ///
3097 /// A runtime this one has no link to is alive here as long as its beat keeps arriving
3098 /// through somebody else, and a runtime with a healthy link is dead here the moment it
3099 /// stops beating. That is what makes exclusivity survive a relay and a crash alike, and
3100 /// it is why this is worth reading beside [`Cooperation::peers`] rather than instead
3101 /// of it.
3102 ///
3103 /// ```
3104 /// # use std::sync::Arc;
3105 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3106 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3107 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3108 /// # use majordomus_cli::mesh::link::HttpTransport;
3109 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3110 /// # use majordomus_cli::mesh::repository::of_root_commits;
3111 /// # let cooperation = Cooperation::new(CooperationSetup {
3112 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3113 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3114 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3115 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3116 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3117 /// # }).unwrap();
3118 /// use majordomus_cli::mesh::journal::StreamLiveness;
3119 ///
3120 /// cooperation.journal().beat_own();
3121 /// let streams = cooperation.streams();
3122 /// let (id, liveness, age) = &streams[0];
3123 ///
3124 /// assert_eq!(id, cooperation.journal().own_stream(), "a runtime hears itself first");
3125 /// assert_eq!(*liveness, StreamLiveness::Own, "never judged against its own clock");
3126 /// assert!(age.is_some(), "and it knows when it last beat");
3127 /// ```
3128 pub fn streams(&self) -> Vec<(StreamId, StreamLiveness, Option<u64>)> {
3129 self.journal
3130 .stream_liveness(self.expiry())
3131 .into_iter()
3132 .map(|(id, (l, age))| (id, l, age.map(|a| a.as_millis() as u64)))
3133 .collect()
3134 }
3135
3136 /// Verify cooperation now: the local checks, then one sync round with every peer this
3137 /// runtime dials, and whether both sides hold the same marks afterwards.
3138 ///
3139 /// It acts rather than reports. Everything else here answers from what happened to
3140 /// arrive; this goes and asks, because the failure it exists to catch is the one where
3141 /// every counter looks reasonable and no traffic is actually crossing. It is therefore
3142 /// safe to run and not free: it writes nothing, but it costs a round per peer.
3143 ///
3144 /// ```
3145 /// # use std::collections::BTreeMap;
3146 /// # use std::sync::{Arc, Mutex, Weak};
3147 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3148 /// # use majordomus_cli::mesh::cooperation::*;
3149 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3150 /// # use majordomus_cli::mesh::link::{LinkReply, LinkTransport, Signed, HELLO_PATH};
3151 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3152 /// # use majordomus_cli::mesh::repository::{of_root_commits, runtime_id};
3153 /// # use majordomus_cli::mesh::trust::TrustPolicy;
3154 /// # #[derive(Default)] struct Net(Mutex<BTreeMap<String, Weak<Cooperation>>>);
3155 /// # impl LinkTransport for Net {
3156 /// # fn post(&self, e: &str, p: &str, m: &Signed) -> Result<LinkReply, String> {
3157 /// # let peer = (self.0.lock().unwrap().get(e).and_then(Weak::upgrade))
3158 /// # .ok_or_else(|| format!("{e}: nobody there"))?;
3159 /// # Ok(if p == HELLO_PATH { peer.accept_hello(m) } else { peer.accept_sync(m) }) } }
3160 /// # let net = Arc::new(Net::default());
3161 /// # let runtime = |endpoint: &str| { let c = Cooperation::new(CooperationSetup {
3162 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: runtime_id(endpoint),
3163 /// # repository: of_root_commits(&["root".into()]), endpoints: vec![endpoint.into()],
3164 /// # version: "doc".into(), config: CooperationConfig::default(),
3165 /// # trust: TrustConfig { policy: TrustPolicy::Tofu, allow: vec![] }, journal_path: None,
3166 /// # registry: Arc::new(MeshRegistry::new()),
3167 /// # transport: Arc::clone(&net) as Arc<dyn LinkTransport>, board: None,
3168 /// # checkout: CheckoutFacts::default() }).unwrap();
3169 /// # net.0.lock().unwrap().insert(endpoint.into(), Arc::downgrade(&c)); c };
3170 /// # let (a, b) = (runtime("a:1"), runtime("b:1"));
3171 /// a.dial(&["b:1".into()]).unwrap();
3172 ///
3173 /// a.journal().beat_own();
3174 /// let report = a.verify();
3175 /// assert!(report.ok, "every check held and every round went through");
3176 /// assert_eq!(report.peers[0].converged, Some(true), "and the two hold the same marks");
3177 /// assert_eq!(report.digest, Some(b.state().digest), "which is one state, seen twice");
3178 /// ```
3179 pub fn verify(&self) -> MeshVerifyReport {
3180 let mut checks = Vec::new();
3181 let heartbeat = self.heartbeat();
3182 checks.push(MeshVerifyCheck {
3183 check: "cooperation".into(),
3184 ok: !self.stopped() && self.config.enabled,
3185 detail: format!(
3186 "runtime {} — heartbeat {}s, expiry {}s, link protocol {LINK_PROTOCOL_MIN}..{LINK_PROTOCOL_MAX}",
3187 self.own_key, self.config.heartbeat_seconds, self.config.expiry_seconds
3188 ),
3189 impact: None,
3190 remediation: None,
3191 });
3192 checks.push(MeshVerifyCheck {
3193 check: "repository".into(),
3194 ok: true,
3195 detail: format!(
3196 "{} from {:?}: {}",
3197 self.repository.id, self.repository.basis, self.repository.detail
3198 ),
3199 impact: None,
3200 remediation: None,
3201 });
3202 let loopback = super::address::loopback_only(&self.card.endpoints);
3203 checks.push(MeshVerifyCheck {
3204 check: "endpoints".into(),
3205 ok: true,
3206 detail: self.card.endpoints.join(", "),
3207 impact: loopback.then(|| {
3208 "only runtimes on this machine can dial this one; this runtime can still dial remote runtimes, and a link works in both directions once dialed".into()
3209 }),
3210 remediation: loopback.then(|| {
3211 "to be dialed from other machines, serve beyond loopback: majordomus serve --host 0.0.0.0 (or a deployment object), on a private network or an overlay".into()
3212 }),
3213 });
3214 let own_age = self
3215 .journal
3216 .stream_liveness(self.expiry())
3217 .get(self.journal.own_stream())
3218 .and_then(|(_, age)| *age);
3219 let beating = own_age.is_some_and(|a| a <= 2 * heartbeat + Duration::from_secs(1));
3220 checks.push(MeshVerifyCheck {
3221 check: "heartbeat".into(),
3222 ok: beating,
3223 detail: match own_age {
3224 Some(a) => format!("this runtime's beat rose {}ms ago", a.as_millis()),
3225 None => "this runtime has not beaten yet".into(),
3226 },
3227 impact: (!beating).then(|| "linked runtimes will expire this one and stop honouring its claims".into()),
3228 remediation: (!beating).then(|| "the cooperation thread is not running: restart the server (majordomus serve stop && majordomus serve ensure)".into()),
3229 });
3230 let tallies = self.journal.tallies();
3231 checks.push(MeshVerifyCheck {
3232 check: "journal".into(),
3233 ok: tallies.pending == 0,
3234 detail: format!(
3235 "{} events in {} streams; {} received, {} duplicates absorbed, {} rejected, {} pending, {} opaque",
3236 tallies.events, tallies.streams, tallies.received, tallies.duplicates, tallies.rejected, tallies.pending, tallies.opaque
3237 ),
3238 impact: (tallies.pending > 0).then(|| "events after a gap are held until the gap arrives; the state lags the stream they belong to".into()),
3239 remediation: (tallies.pending > 0).then(|| "a peer relays a partial stream: run mesh verify on the peers and compare their marks".into()),
3240 });
3241 let refused = self.status().refused;
3242 let suspicious: Vec<&RefusedView> = refused
3243 .iter()
3244 .filter(|r| {
3245 matches!(
3246 r.refusal.code,
3247 RefusalCode::Untrusted
3248 | RefusalCode::ProtocolUnsupported
3249 | RefusalCode::Signature
3250 )
3251 })
3252 .collect();
3253 checks.push(MeshVerifyCheck {
3254 check: "refusals".into(),
3255 // A runtime of this repository refused for trust, protocol or signature is a
3256 // misconfiguration or an attack, never a healthy mesh. A repository mismatch is
3257 // isolation working and does not fail the verification.
3258 ok: suspicious.is_empty(),
3259 detail: if refused.is_empty() {
3260 "no candidate refused".into()
3261 } else {
3262 refused
3263 .iter()
3264 .map(|r| format!("{} {}: {}", r.endpoint, r.refusal.code.as_str(), r.refusal.detail))
3265 .collect::<Vec<_>>()
3266 .join("; ")
3267 },
3268 impact: (!suspicious.is_empty()).then(|| {
3269 "a runtime of this repository may be refused for trust, protocol or signature: it cooperates with nobody here".into()
3270 }),
3271 remediation: (!suspicious.is_empty()).then(|| {
3272 "untrusted: add its key (majordomus mesh identity on that machine) to trust.allow; protocol: upgrade the older executable; signature: that peer's identity file changed or the traffic is tampered".into()
3273 }),
3274 });
3275
3276 let mut peers = Vec::new();
3277 for peer in self.peers() {
3278 if peer.outbound {
3279 let outcome = self.sync_with(&peer.runtime);
3280 let after = self.peers().into_iter().find(|p| p.runtime == peer.runtime);
3281 let remote = {
3282 let table = self.table.lock().expect("cooperation table");
3283 table
3284 .peers
3285 .get(&peer.runtime)
3286 .and_then(|p| p.outbound.as_ref())
3287 .map(|o| o.remote_marks.clone())
3288 };
3289 let converged = remote.map(|remote| {
3290 let local = self.journal.marks();
3291 local.keys().chain(remote.keys()).all(|s| {
3292 local.get(s).map(|m| m.seq).unwrap_or(0)
3293 == remote.get(s).map(|m| m.seq).unwrap_or(0)
3294 })
3295 });
3296 peers.push(PeerVerdict {
3297 runtime: peer.runtime.clone(),
3298 state: after
3299 .as_ref()
3300 .map(|p| p.state)
3301 .unwrap_or(LinkState::Expired),
3302 dialed: true,
3303 round_trip: Some(outcome.is_ok()),
3304 rtt_ms: after.and_then(|p| p.rtt_ms),
3305 converged,
3306 detail: match outcome {
3307 Ok(()) => "one sync round, signed both ways".into(),
3308 Err(e) => e.to_string(),
3309 },
3310 });
3311 } else {
3312 let fresh = peer.state == LinkState::Connected;
3313 peers.push(PeerVerdict {
3314 runtime: peer.runtime.clone(),
3315 state: peer.state,
3316 dialed: false,
3317 round_trip: None,
3318 rtt_ms: peer.rtt_ms,
3319 converged: None,
3320 detail: match peer.last_exchange_ms {
3321 Some(ms) if fresh => {
3322 format!("the peer dials this runtime; last exchange {ms}ms ago")
3323 }
3324 Some(ms) => {
3325 format!("the peer dials this runtime and has been silent {ms}ms")
3326 }
3327 None => "the peer has not exchanged anything yet".into(),
3328 },
3329 });
3330 }
3331 }
3332 // Declared seeds say that peers exist: a runtime with seeds and no connected peer has
3333 // verified nothing, however clean its local checks are.
3334 let connected = peers
3335 .iter()
3336 .filter(|p| p.state == LinkState::Connected)
3337 .count();
3338 let expected = self.config.seeds.len();
3339 let starved = expected > 0 && connected == 0;
3340 checks.push(MeshVerifyCheck {
3341 check: "links".into(),
3342 ok: !starved,
3343 detail: format!(
3344 "{connected} connected of {} linked peer(s); {expected} seed(s) declared",
3345 peers.len()
3346 ),
3347 impact: starved.then(|| {
3348 "the seeds this repository declares cooperate with nobody here: no claim, handover or review crosses".into()
3349 }),
3350 remediation: starved.then(|| {
3351 "read the refusals above; an unreachable seed is a network or bind problem — is the peer served beyond loopback?".into()
3352 }),
3353 });
3354 // An expired peer is listed with its state and does not fail the verification: a
3355 // peer that shut down cleanly is gone, not broken. A round that fails does.
3356 let ok = checks.iter().all(|c| c.ok)
3357 && peers
3358 .iter()
3359 .all(|p| p.state == LinkState::Expired || p.round_trip != Some(false));
3360 MeshVerifyReport {
3361 ok,
3362 runtime: Some(self.own_key.clone()),
3363 checks,
3364 peers,
3365 digest: Some(self.state().digest),
3366 }
3367 }
3368
3369 // ------------------------------------------------------------------ operations
3370
3371 fn write(&self, body: EventBody, key: String) -> Result<Written, CooperationError> {
3372 let appended = self
3373 .journal
3374 .append_acknowledged(body)
3375 .map_err(|e| CooperationError::Invalid(e.to_string()))?;
3376 Ok(Written {
3377 key,
3378 event: appended.event.id(),
3379 lamport: appended.event.lamport,
3380 durability: appended.durability,
3381 })
3382 }
3383
3384 fn own(&self, local: &str) -> String {
3385 format!("{}/{local}", self.journal.own_stream())
3386 }
3387
3388 fn session_exists(&self, state: &CooperationState, session: &str) -> bool {
3389 let key = self.own(session);
3390 state.sessions.iter().any(|s| s.key == key)
3391 }
3392
3393 /// Open a session of this runtime, or say again what an open one is doing. One call
3394 /// serves both because a session is identified by its name and described by the rest:
3395 /// announcing a new intent under an existing name is an update every runtime folds the
3396 /// same way, and nothing has to be told apart from what came before.
3397 ///
3398 /// ```
3399 /// # use std::sync::Arc;
3400 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3401 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3402 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3403 /// # use majordomus_cli::mesh::link::HttpTransport;
3404 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3405 /// # use majordomus_cli::mesh::repository::of_root_commits;
3406 /// # let cooperation = Cooperation::new(CooperationSetup {
3407 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3408 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3409 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3410 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3411 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3412 /// # }).unwrap();
3413 /// use majordomus_cli::mesh::journal::SessionInfo;
3414 ///
3415 /// let opened = cooperation.open_session(SessionInfo::named("s1", "claude-code")).unwrap();
3416 /// assert!(opened.key.ends_with("/s1"), "qualified by the stream that wrote it");
3417 ///
3418 /// let with_intent = SessionInfo { intent: Some("documenting the mesh".into()),
3419 /// ..SessionInfo::named("s1", "claude-code") };
3420 /// cooperation.open_session(with_intent).unwrap();
3421 ///
3422 /// let sessions = cooperation.state().sessions;
3423 /// assert_eq!(sessions.len(), 1, "said again, not opened twice");
3424 /// assert_eq!(sessions[0].info.intent.as_deref(), Some("documenting the mesh"));
3425 /// ```
3426 pub fn open_session(&self, info: SessionInfo) -> Result<Written, CooperationError> {
3427 let key = self.own(&info.session);
3428 self.write(EventBody::SessionOpened { info }, key)
3429 }
3430
3431 /// Close a session of this runtime; its claims end with it.
3432 ///
3433 /// Claims end with the session rather than having to be released one by one, because
3434 /// the commonest way for a worker to stop is not an orderly one: what a session held
3435 /// must be recoverable from the fact that it is over. Only this runtime may close its
3436 /// own sessions — a session on another runtime is closed by that runtime, or expires
3437 /// when it stops beating.
3438 ///
3439 /// ```
3440 /// # use std::sync::Arc;
3441 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3442 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3443 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3444 /// # use majordomus_cli::mesh::link::HttpTransport;
3445 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3446 /// # use majordomus_cli::mesh::repository::of_root_commits;
3447 /// # let cooperation = Cooperation::new(CooperationSetup {
3448 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3449 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3450 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3451 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3452 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3453 /// # }).unwrap();
3454 /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3455 /// use majordomus_cli::mesh::state::SessionState;
3456 ///
3457 /// let session = SessionInfo::named("s1", "cli");
3458 /// cooperation.claim(&session, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
3459 /// cooperation.close_session("s1").unwrap();
3460 ///
3461 /// let state = cooperation.state();
3462 /// assert_eq!(state.sessions[0].state, SessionState::Closed);
3463 /// assert!(!state.claims[0].state.is_live(), "nobody had to release it");
3464 ///
3465 /// // and a name this runtime never opened is not something it can close
3466 /// cooperation.close_session("s9").expect_err("no such session here");
3467 /// ```
3468 pub fn close_session(&self, session: &str) -> Result<Written, CooperationError> {
3469 let state = self.state();
3470 if !self.session_exists(&state, session) {
3471 return Err(CooperationError::NotFound(format!(
3472 "no session '{session}' on this runtime"
3473 )));
3474 }
3475 self.write(
3476 EventBody::SessionClosed {
3477 session: session.into(),
3478 },
3479 self.own(session),
3480 )
3481 }
3482
3483 /// Claim a scope for a session of this runtime. An exclusive claim that meets a live
3484 /// exclusive claim of another session — on any runtime this one has heard — is
3485 /// refused with the claims it meets; the session is opened first if it is new.
3486 ///
3487 /// This is where exclusion actually happens. Admission is checked against the folded
3488 /// state of every runtime this one has heard from, under a lock, so two callers here
3489 /// cannot both be told yes; and an advisory claim is never refused, because its purpose
3490 /// is to make two workers visible to each other rather than to stop either.
3491 ///
3492 /// ```
3493 /// # use std::sync::Arc;
3494 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3495 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3496 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3497 /// # use majordomus_cli::mesh::link::HttpTransport;
3498 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3499 /// # use majordomus_cli::mesh::repository::of_root_commits;
3500 /// # let cooperation = Cooperation::new(CooperationSetup {
3501 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3502 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3503 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3504 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3505 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3506 /// # }).unwrap();
3507 /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3508 ///
3509 /// let first = SessionInfo::named("s1", "cli");
3510 /// cooperation.claim(&first, vec!["apps/majordomus-cli".into()],
3511 /// Some("the mesh module".into()), ClaimMode::Exclusive, Some("#184".into())).unwrap();
3512 ///
3513 /// // a scope under a held one is the same place, and another session is refused it
3514 /// let second = SessionInfo::named("s2", "cli");
3515 /// cooperation.claim(&second, vec!["apps/majordomus-cli/src".into()], None,
3516 /// ClaimMode::Exclusive, None).expect_err("the scope is claimed");
3517 ///
3518 /// // announcing the same work advisedly is always allowed: it informs, it does not exclude
3519 /// cooperation.claim(&second, vec!["apps/majordomus-cli/src".into()], None,
3520 /// ClaimMode::Advisory, None).unwrap();
3521 /// assert_eq!(cooperation.state().overlaps.len(), 1, "two workers, told about each other");
3522 /// ```
3523 pub fn claim(
3524 &self,
3525 session: &SessionInfo,
3526 scope: Vec<String>,
3527 intent: Option<String>,
3528 mode: ClaimMode,
3529 issue: Option<String>,
3530 ) -> Result<Written, CooperationError> {
3531 // Check and append under one lock: two callers of this runtime must not both pass
3532 // admission for one scope and leave the fold to name a conflict it could prevent.
3533 let _admission = self.admission.lock().expect("claim admission");
3534 let state = self.state();
3535 let conflicts = admission_conflicts(&state, &self.own(&session.session), &scope, mode);
3536 if !conflicts.is_empty() {
3537 self.counters.claims_refused.fetch_add(1, Ordering::Relaxed);
3538 let conflicts: Vec<ClaimView> = conflicts.into_iter().cloned().collect();
3539 tracing::info!(runtime_id = %self.own_key, session_id = %session.session, conflicts = conflicts.len(), "mesh claim refused: the scope is claimed");
3540 return Err(CooperationError::Conflict(conflicts));
3541 }
3542 if !self.session_exists(&state, &session.session) {
3543 self.open_session(session.clone())?;
3544 }
3545 let claim = format!("c-{}", &fresh_token()[..12]);
3546 self.write(
3547 EventBody::ClaimAcquired {
3548 claim: claim.clone(),
3549 session: session.session.clone(),
3550 scope,
3551 intent,
3552 mode,
3553 issue,
3554 },
3555 self.own(&claim),
3556 )
3557 }
3558
3559 /// Release a claim this runtime took in its current run.
3560 ///
3561 /// Only the holder releases: a claim is a statement by one runtime about what it is
3562 /// doing, and letting another runtime withdraw it would make the record say something
3563 /// its author never said. A dead holder is not a problem this needs to solve — its
3564 /// stream stops beating and its claims stop excluding on every runtime at once.
3565 ///
3566 /// ```
3567 /// # use std::sync::Arc;
3568 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3569 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3570 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3571 /// # use majordomus_cli::mesh::link::HttpTransport;
3572 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3573 /// # use majordomus_cli::mesh::repository::of_root_commits;
3574 /// # let cooperation = Cooperation::new(CooperationSetup {
3575 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3576 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3577 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3578 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3579 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3580 /// # }).unwrap();
3581 /// use majordomus_cli::mesh::journal::{ClaimMode, SessionInfo};
3582 ///
3583 /// let mine = SessionInfo::named("s1", "cli");
3584 /// let held = cooperation
3585 /// .claim(&mine, vec!["apps".into()], None, ClaimMode::Exclusive, None)
3586 /// .unwrap();
3587 /// cooperation.release(&held.key).unwrap();
3588 ///
3589 /// // the scope is free again, for anybody
3590 /// let other = SessionInfo::named("s2", "cli");
3591 /// cooperation.claim(&other, vec!["apps".into()], None, ClaimMode::Exclusive, None).unwrap();
3592 ///
3593 /// // a claim of another runtime is not this one's to withdraw
3594 /// cooperation.release("somebody-0000000000000002/c-abc").expect_err("not its author");
3595 /// ```
3596 pub fn release(&self, claim_key: &str) -> Result<Written, CooperationError> {
3597 let own_prefix = format!("{}/", self.journal.own_stream());
3598 let Some(local) = claim_key.strip_prefix(&own_prefix) else {
3599 return Err(CooperationError::NotOwn(format!(
3600 "{claim_key} was not claimed by this runtime's current run; only its holder releases a claim, and a dead holder's claim expires on its own"
3601 )));
3602 };
3603 let state = self.state();
3604 if !state.claims.iter().any(|c| c.key == claim_key) {
3605 return Err(CooperationError::NotFound(format!("no claim {claim_key}")));
3606 }
3607 self.write(
3608 EventBody::ClaimReleased {
3609 claim: local.into(),
3610 },
3611 claim_key.into(),
3612 )
3613 }
3614
3615 /// Publish a handover for every linked runtime to pick up.
3616 ///
3617 /// A handover is identified by the digest of what it says, so publishing the same
3618 /// continuity twice — by a retry, or by two runtimes that both saw it — leaves one
3619 /// handover rather than a choice between duplicates for whoever comes to take it.
3620 ///
3621 /// ```
3622 /// # use std::sync::Arc;
3623 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3624 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3625 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3626 /// # use majordomus_cli::mesh::link::HttpTransport;
3627 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3628 /// # use majordomus_cli::mesh::repository::of_root_commits;
3629 /// # let cooperation = Cooperation::new(CooperationSetup {
3630 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3631 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3632 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3633 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3634 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3635 /// # }).unwrap();
3636 /// use majordomus_cli::mesh::journal::HandoverBody;
3637 ///
3638 /// let body = "# Objective\nfinish the mesh docs\n".to_string();
3639 /// let handover = HandoverBody { id: HandoverBody::digest_of(&body), task: None, issue: None,
3640 /// milestone: None, branch: Some("feature/mesh".into()), head: None, created_at: None,
3641 /// name: None, body };
3642 ///
3643 /// let written = cooperation.publish_handover(handover.clone()).unwrap();
3644 /// assert_eq!(written.key, handover.id, "addressed by what it says");
3645 ///
3646 /// cooperation.publish_handover(handover).unwrap();
3647 /// assert_eq!(cooperation.state().handovers.len(), 1, "published twice, and still one");
3648 /// ```
3649 pub fn publish_handover(&self, handover: HandoverBody) -> Result<Written, CooperationError> {
3650 let id = handover.id.clone();
3651 self.write(EventBody::HandoverPublished { handover }, id)
3652 }
3653
3654 /// Take a handover: return its body and record, for everyone, that this session took
3655 /// it. The record is the point. Two workers picking up the same continuity is the
3656 /// failure this prevents, and it can only be prevented by the taking being visible —
3657 /// so reading a handover and consuming it are one operation, and a second worker sees
3658 /// who was there before it decides.
3659 ///
3660 /// Taking one already taken is not refused, because a handover may legitimately be
3661 /// picked up again; the record simply names everybody who has.
3662 ///
3663 /// ```
3664 /// # use std::sync::Arc;
3665 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3666 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3667 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3668 /// # use majordomus_cli::mesh::link::HttpTransport;
3669 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3670 /// # use majordomus_cli::mesh::repository::of_root_commits;
3671 /// # let cooperation = Cooperation::new(CooperationSetup {
3672 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3673 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3674 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3675 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3676 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3677 /// # }).unwrap();
3678 /// use majordomus_cli::mesh::journal::HandoverBody;
3679 ///
3680 /// let body = "# Objective\nfinish the mesh docs\n".to_string();
3681 /// let id = HandoverBody::digest_of(&body);
3682 /// cooperation.publish_handover(HandoverBody { id: id.clone(), task: None, issue: None,
3683 /// milestone: None, branch: None, head: None, created_at: None, name: None, body }).unwrap();
3684 ///
3685 /// let (taken, _written) = cooperation.consume_handover(&id, "s1").unwrap();
3686 /// assert!(taken.handover.body.contains("mesh docs"), "the continuity itself");
3687 /// assert_eq!(cooperation.state().handovers[0].consumed_by.len(), 1, "and who took it");
3688 ///
3689 /// cooperation.consume_handover("no-such-handover", "s1").expect_err("nothing to take");
3690 /// ```
3691 pub fn consume_handover(
3692 &self,
3693 id: &str,
3694 session: &str,
3695 ) -> Result<(HandoverView, Written), CooperationError> {
3696 let state = self.state();
3697 let Some(view) = state.handovers.into_iter().find(|h| h.id == id) else {
3698 return Err(CooperationError::NotFound(format!(
3699 "no handover {id} in this runtime's journal"
3700 )));
3701 };
3702 let written = self.write(
3703 EventBody::HandoverConsumed {
3704 handover: id.into(),
3705 session: session.into(),
3706 },
3707 id.into(),
3708 )?;
3709 Ok((view, written))
3710 }
3711
3712 /// Ask for a review; a named reviewer must be a linked peer carrying `reviews`.
3713 ///
3714 /// A request may name nobody, and then it is an offer to the whole mesh — which is the
3715 /// useful shape when a worker wants a second opinion and does not care whose. Naming a
3716 /// reviewer is checked here rather than left to fail silently later: a request
3717 /// addressed to a runtime that is not linked, or to one whose executable does not do
3718 /// reviews, would otherwise sit unanswered and look like indifference.
3719 ///
3720 /// ```
3721 /// # use std::sync::Arc;
3722 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3723 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3724 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3725 /// # use majordomus_cli::mesh::link::HttpTransport;
3726 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3727 /// # use majordomus_cli::mesh::repository::of_root_commits;
3728 /// # let cooperation = Cooperation::new(CooperationSetup {
3729 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3730 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3731 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3732 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3733 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3734 /// # }).unwrap();
3735 /// use majordomus_cli::mesh::state::ReviewState;
3736 ///
3737 /// cooperation.request_review("s1", "feature/mesh-cooperation".into(),
3738 /// vec!["apps/majordomus-cli/src/mesh".into()], Some("#184".into()), None, None).unwrap();
3739 ///
3740 /// let review = &cooperation.state().reviews[0];
3741 /// assert_eq!(review.subject, "feature/mesh-cooperation");
3742 /// assert_eq!(review.state, ReviewState::Open, "asked of anybody, answered by nobody yet");
3743 ///
3744 /// // a reviewer nobody is linked to is refused now, not left waiting
3745 /// cooperation.request_review("s1", "feature/x".into(), vec![], None,
3746 /// Some("somebody-0000000000000002".into()), None).expect_err("no such peer");
3747 /// ```
3748 pub fn request_review(
3749 &self,
3750 session: &str,
3751 subject: String,
3752 scope: Vec<String>,
3753 issue: Option<String>,
3754 reviewer: Option<String>,
3755 body: Option<String>,
3756 ) -> Result<Written, CooperationError> {
3757 if let Some(target) = &reviewer {
3758 if *target != self.own_key {
3759 let table = self.table.lock().expect("cooperation table");
3760 match table.peers.get(target) {
3761 None => {
3762 return Err(CooperationError::NotFound(format!(
3763 "no linked peer {target}"
3764 )))
3765 }
3766 Some(peer) if !peer.card.supports("reviews") => {
3767 return Err(CooperationError::FeatureUnsupported(format!(
3768 "{target} does not carry the reviews feature (it carries: {})",
3769 peer.card.features.join(", ")
3770 )))
3771 }
3772 Some(_) => {}
3773 }
3774 }
3775 }
3776 let review = format!("r-{}", &fresh_token()[..12]);
3777 self.write(
3778 EventBody::ReviewRequested {
3779 review: review.clone(),
3780 session: session.into(),
3781 subject: redacted(subject),
3782 scope,
3783 issue,
3784 reviewer,
3785 body: body.map(redacted),
3786 },
3787 self.own(&review),
3788 )
3789 }
3790
3791 /// Answer a review request — anybody's, on any runtime, except the session that asked.
3792 ///
3793 /// The one refusal here is self-review, because a verdict from the session under review
3794 /// is not a second pair of eyes and a record that allowed it would make every review in
3795 /// the mesh worth less. Several sessions may answer one request and their verdicts all
3796 /// stand: the disagreement is the information.
3797 ///
3798 /// ```
3799 /// # use std::sync::Arc;
3800 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3801 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3802 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3803 /// # use majordomus_cli::mesh::link::HttpTransport;
3804 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3805 /// # use majordomus_cli::mesh::repository::of_root_commits;
3806 /// # let cooperation = Cooperation::new(CooperationSetup {
3807 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3808 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3809 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3810 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3811 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3812 /// # }).unwrap();
3813 /// use majordomus_cli::mesh::state::ReviewState;
3814 ///
3815 /// let asked = cooperation
3816 /// .request_review("s1", "feature/mesh-cooperation".into(), vec![], None, None, None)
3817 /// .unwrap();
3818 ///
3819 /// cooperation.answer_review(&asked.key, "s2", "approved".into(),
3820 /// Some("the examples run".into())).unwrap();
3821 /// let review = &cooperation.state().reviews[0];
3822 /// assert_eq!(review.state, ReviewState::Answered);
3823 /// assert_eq!(review.answers[0].verdict, "approved");
3824 ///
3825 /// // the session under review does not get a vote on itself
3826 /// cooperation.answer_review(&asked.key, "s1", "approved".into(), None)
3827 /// .expect_err("a review is another session's verdict");
3828 /// ```
3829 pub fn answer_review(
3830 &self,
3831 request: &str,
3832 session: &str,
3833 verdict: String,
3834 note: Option<String>,
3835 ) -> Result<Written, CooperationError> {
3836 let state = self.state();
3837 let Some(review) = state.reviews.iter().find(|r| r.key == request) else {
3838 return Err(CooperationError::NotFound(format!(
3839 "no review request {request}"
3840 )));
3841 };
3842 if review.session == self.own(session) {
3843 return Err(CooperationError::Invalid(format!(
3844 "session {session} requested review {request} and cannot answer it: a review is another session's verdict"
3845 )));
3846 }
3847 self.write(
3848 EventBody::ReviewAnswered {
3849 request: request.into(),
3850 session: session.into(),
3851 verdict: redacted(verdict),
3852 note: note.map(redacted),
3853 },
3854 request.into(),
3855 )
3856 }
3857
3858 /// End a review request this runtime asked for: done, void or superseded, with an
3859 /// optional result the request then carries. Any session of the asking runtime may —
3860 /// an MCP session is one connection, and the asker that reconnects is another session
3861 /// of the same runtime — and no other runtime may: a withdrawal is the requester's own
3862 /// record, never a verdict, so it does not count as an answer.
3863 ///
3864 /// ```
3865 /// # use std::sync::Arc;
3866 /// # use majordomus_cli::mesh::config::{CooperationConfig, TrustConfig};
3867 /// # use majordomus_cli::mesh::cooperation::{CheckoutFacts, Cooperation, CooperationSetup};
3868 /// # use majordomus_cli::mesh::identity::NodeIdentity;
3869 /// # use majordomus_cli::mesh::link::HttpTransport;
3870 /// # use majordomus_cli::mesh::registry::MeshRegistry;
3871 /// # use majordomus_cli::mesh::repository::of_root_commits;
3872 /// # let cooperation = Cooperation::new(CooperationSetup {
3873 /// # identity: Arc::new(NodeIdentity::ephemeral().unwrap()), runtime: "0000000000000001".into(),
3874 /// # repository: of_root_commits(&["root".into()]), endpoints: vec!["127.0.0.1:9".into()],
3875 /// # version: "doc".into(), config: CooperationConfig::default(), trust: TrustConfig::default(),
3876 /// # journal_path: None, registry: Arc::new(MeshRegistry::new()),
3877 /// # transport: Arc::new(HttpTransport), board: None, checkout: CheckoutFacts::default(),
3878 /// # }).unwrap();
3879 /// use majordomus_cli::mesh::state::ReviewState;
3880 ///
3881 /// let asked = cooperation
3882 /// .request_review("s1", "pre-flight of #919".into(), vec![], None, None,
3883 /// Some("structure job and own cases, hook environment unset".into()))
3884 /// .unwrap();
3885 ///
3886 /// // a request of another runtime cannot be ended here
3887 /// cooperation.withdraw_review("other-0000000000000009/r-1", "s1", None)
3888 /// .expect_err("not this runtime's request");
3889 ///
3890 /// cooperation.withdraw_review(&asked.key, "s1", Some("clean: 45 gates, 12 cases".into()))
3891 /// .unwrap();
3892 /// let review = &cooperation.state().reviews[0];
3893 /// assert_eq!(review.state, ReviewState::Withdrawn);
3894 /// assert_eq!(review.result.as_deref(), Some("clean: 45 gates, 12 cases"));
3895 /// assert!(review.answers.is_empty(), "a result is not an answer");
3896 /// ```
3897 pub fn withdraw_review(
3898 &self,
3899 request: &str,
3900 session: &str,
3901 note: Option<String>,
3902 ) -> Result<Written, CooperationError> {
3903 let state = self.state();
3904 let Some(review) = state.reviews.iter().find(|r| r.key == request) else {
3905 return Err(CooperationError::NotFound(format!(
3906 "no review request {request}"
3907 )));
3908 };
3909 if !review
3910 .key
3911 .starts_with(&format!("{}/", self.journal.own_stream()))
3912 {
3913 return Err(CooperationError::Invalid(format!(
3914 "review {request} was asked by {}, and only the runtime that asked may withdraw it",
3915 review.session
3916 )));
3917 }
3918 self.write(
3919 EventBody::ReviewWithdrawn {
3920 request: request.into(),
3921 session: session.into(),
3922 note: note.map(redacted),
3923 },
3924 request.into(),
3925 )
3926 }
3927}
3928
3929/// Free text of a review as it is signed: every credential shape replaced by the marker that
3930/// names it, through the one rule set (`redaction::redact_secrets`), as a handover body is.
3931///
3932/// A review's subject, body, verdict and notes are typed by a session and replicated in clear
3933/// to every linked runtime, so a pasted credential would otherwise travel with them (I2708).
3934/// The entries of `.ai/repo/ci/mesh-event-fields.yaml` that say `redacted` name these fields.
3935fn redacted(text: String) -> String {
3936 crate::redaction::redact_secrets(&text).text
3937}
3938
3939/// A JSON error as the text a refusal or a round error carries.
3940fn json_text(error: serde_json::Error) -> String {
3941 error.to_string()
3942}
3943
3944#[cfg(test)]
3945mod tests {
3946 use super::*;
3947 use crate::mesh::repository::of_root_commits;
3948 use crate::mesh::trust::TrustPolicy;
3949 use std::sync::Weak;
3950
3951 /// Two or more runtimes in one process, linked through this transport: the protocol
3952 /// under test is the real one; only the socket is replaced. Real sockets are the
3953 /// integration tests' (tests/mesh_cooperation.rs).
3954 #[derive(Default)]
3955 struct InProcess {
3956 runtimes: Mutex<BTreeMap<String, Weak<Cooperation>>>,
3957 cut: Mutex<Vec<String>>,
3958 /// Run after a peer has answered a sync and before the answer reaches the dialer:
3959 /// the window a slow machine stretches, held still.
3960 answered: Mutex<Option<Answered>>,
3961 }
3962
3963 type Answered = Box<dyn Fn() + Send>;
3964
3965 /// The one trust definition the live path and the reload share: this machine's own key
3966 /// is trusted, and a value that is not a key is rejected before any policy is asked —
3967 /// whatever the policy, even one that trusts on first use.
3968 #[test]
3969 fn a_value_that_is_not_a_key_is_rejected_by_every_policy() {
3970 let registry = MeshRegistry::new();
3971 let own = NodeIdentity::ephemeral().unwrap().public.public_key.clone();
3972 for policy in [TrustPolicy::Tofu, TrustPolicy::DenyUnknown] {
3973 let config = TrustConfig {
3974 policy,
3975 allow: vec!["not-a-key".into()],
3976 };
3977 assert!(key_trust(&own, &config, ®istry, &own).is_trusted());
3978 assert_eq!(
3979 key_trust(&own, &config, ®istry, "not-a-key"),
3980 TrustState::Rejected("not a key".into())
3981 );
3982 }
3983 }
3984
3985 impl LinkTransport for InProcess {
3986 fn post(&self, endpoint: &str, path: &str, message: &Signed) -> Result<LinkReply, String> {
3987 if self.cut.lock().unwrap().iter().any(|c| c == endpoint) {
3988 return Err(format!("{endpoint}: partitioned"));
3989 }
3990 let target = self
3991 .runtimes
3992 .lock()
3993 .unwrap()
3994 .get(endpoint)
3995 .and_then(Weak::upgrade)
3996 .ok_or_else(|| format!("{endpoint}: connection refused"))?;
3997 let reply = match path {
3998 HELLO_PATH => target.accept_hello(message),
3999 SYNC_PATH => target.accept_sync(message),
4000 other => return Err(format!("no route {other}")),
4001 };
4002 if path == SYNC_PATH {
4003 if let Some(answered) = self.answered.lock().unwrap().as_ref() {
4004 answered();
4005 }
4006 }
4007 Ok(reply)
4008 }
4009 }
4010
4011 fn runtime(
4012 net: &Arc<InProcess>,
4013 endpoint: &str,
4014 repo: &str,
4015 policy: TrustPolicy,
4016 ) -> Arc<Cooperation> {
4017 let identity = Arc::new(NodeIdentity::ephemeral().unwrap());
4018 let c = Cooperation::new(CooperationSetup {
4019 identity,
4020 runtime: crate::mesh::repository::runtime_id(endpoint),
4021 repository: of_root_commits(&[repo.into()]),
4022 endpoints: vec![endpoint.into()],
4023 version: "test".into(),
4024 config: CooperationConfig {
4025 heartbeat_seconds: 1,
4026 expiry_seconds: 3,
4027 ..CooperationConfig::default()
4028 },
4029 trust: TrustConfig {
4030 policy,
4031 allow: vec![],
4032 },
4033 journal_path: None,
4034 registry: Arc::new(MeshRegistry::new()),
4035 transport: Arc::clone(net) as Arc<dyn LinkTransport>,
4036 board: None,
4037 checkout: CheckoutFacts::default(),
4038 })
4039 .unwrap();
4040 net.runtimes
4041 .lock()
4042 .unwrap()
4043 .insert(endpoint.into(), Arc::downgrade(&c));
4044 c
4045 }
4046
4047 fn session(name: &str) -> SessionInfo {
4048 SessionInfo::named(name, "test")
4049 }
4050
4051 #[test]
4052 fn retrust_ends_a_link_this_runtime_dialed_as_well_as_one_it_answered() {
4053 let net = Arc::new(InProcess::default());
4054 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4055 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4056 b.dial(&["a:1".into()]).expect("linked under tofu");
4057 // B dialed: it holds A outbound only, and A's key loses its trust all the same
4058 let ended = b.retrust(TrustConfig::default());
4059 assert_eq!(ended, vec![a.runtime_key().to_string()]);
4060 let synced = b.sync_with(a.runtime_key());
4061 assert!(matches!(synced, Err(RoundError::NoLink)), "{synced:?}");
4062 }
4063
4064 /// A body whose serialization fails once it carries an event.
4065 struct RefusesEvents(usize);
4066
4067 impl Serialize for RefusesEvents {
4068 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
4069 if self.0 > 0 {
4070 return Err(serde::ser::Error::custom("no events here"));
4071 }
4072 serializer.serialize_unit()
4073 }
4074 }
4075
4076 #[test]
4077 fn a_sync_message_over_the_bound_or_unwritable_is_an_error_and_a_heavy_one_drops_events() {
4078 let net = Arc::new(InProcess::default());
4079 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4080 for name in ["s1", "s2"] {
4081 a.open_session(session(name)).unwrap();
4082 }
4083 let nothing = Marks::new();
4084 let over = a.sync_within_bound(Domain::SyncRequest, ¬hing, |_| {
4085 "x".repeat(MAX_LINK_MESSAGE)
4086 });
4087 assert!(over.unwrap_err().contains("link bound"), "no event fits");
4088 let refused = a.sync_within_bound(Domain::SyncRequest, ¬hing, |events| {
4089 RefusesEvents(events.len())
4090 });
4091 assert_eq!(refused.unwrap_err(), "no events here");
4092 let bare = a.sync_within_bound(Domain::SyncRequest, ¬hing, |_| RefusesEvents(1));
4093 assert_eq!(
4094 bare.unwrap_err(),
4095 "no events here",
4096 "not even the bare message"
4097 );
4098 // a message that grows faster than its events loses events from the end until it fits
4099 let half = MAX_LINK_MESSAGE / 2;
4100 let (_, kept) = a
4101 .sync_within_bound(Domain::SyncRequest, ¬hing, |events| {
4102 "x".repeat(events.len() * half)
4103 })
4104 .unwrap();
4105 assert_eq!(kept, 1, "one event fits beside its padding");
4106 let (bare, kept) = a
4107 .sync_within_bound(Domain::SyncRequest, ¬hing, |events| {
4108 "x".repeat(events.len() * MAX_LINK_MESSAGE)
4109 })
4110 .unwrap();
4111 assert_eq!(kept, 0, "not one event fits, and the bare message goes");
4112 assert_eq!(bare.body, serde_json::json!(""));
4113 }
4114
4115 #[test]
4116 fn a_sync_over_the_link_bound_is_unreachable_and_an_answer_over_it_is_refused() {
4117 let net = Arc::new(InProcess::default());
4118 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4119 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4120 let key = a.dial(&["b:1".into()]).unwrap();
4121 a.sync_with(&key).unwrap();
4122 b.open_session(session("s1")).unwrap();
4123 let link = {
4124 let mut table = a.table.lock().unwrap();
4125 let out = table
4126 .peers
4127 .get_mut(&key)
4128 .unwrap()
4129 .outbound
4130 .as_mut()
4131 .unwrap();
4132 std::mem::replace(&mut out.link, "x".repeat(MAX_LINK_MESSAGE))
4133 };
4134 let synced = a.sync_with(&key);
4135 assert!(
4136 matches!(synced, Err(RoundError::Unreachable(ref e)) if e.contains("link bound")),
4137 "{synced:?}"
4138 );
4139 // a request exactly at the bound whose answer, carrying B's marks, is over it
4140 let request = |link: String| SyncRequest {
4141 link,
4142 counter: 1_000_000,
4143 instance: a.card.instance.clone(),
4144 marks: Marks::new(),
4145 events: vec![],
4146 };
4147 let signed = |r: &SyncRequest| {
4148 sign(
4149 &a.identity,
4150 Domain::SyncRequest,
4151 serde_json::to_value(r).unwrap(),
4152 )
4153 };
4154 let base = serde_json::to_vec(&signed(&request(String::new())))
4155 .unwrap()
4156 .len();
4157 let long = "y".repeat(MAX_LINK_MESSAGE - base);
4158 let message = signed(&request(long.clone()));
4159 assert_eq!(
4160 serde_json::to_vec(&message).unwrap().len(),
4161 MAX_LINK_MESSAGE
4162 );
4163 {
4164 let mut table = b.table.lock().unwrap();
4165 let peer = table.inbound_links[&link].clone();
4166 table.inbound_links.insert(long, peer);
4167 }
4168 let reply = b.accept_sync(&message);
4169 assert_eq!(
4170 reply.refusal.map(|r| r.code),
4171 Some(RefusalCode::Oversized),
4172 "the answer cannot be built under the bound"
4173 );
4174 }
4175
4176 #[test]
4177 fn a_round_s_outcome_sets_the_pause_before_the_next() {
4178 let net = Arc::new(InProcess::default());
4179 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4180 let refusal = |code| {
4181 Err(RoundError::Refused(LinkRefusal {
4182 code,
4183 detail: "test".into(),
4184 proto_min: LINK_PROTOCOL_MIN,
4185 proto_max: LINK_PROTOCOL_MAX,
4186 }))
4187 };
4188 let (mut failures, mut unknown) = (0u32, 0u32);
4189 let beat = a.heartbeat();
4190 let unknown_link = refusal(RefusalCode::UnknownLink);
4191 assert_eq!(
4192 a.after_round(&unknown_link, &mut failures, &mut unknown),
4193 Duration::ZERO,
4194 "a restarted answerer is greeted again at once"
4195 );
4196 assert_eq!(
4197 a.after_round(&unknown_link, &mut failures, &mut unknown),
4198 beat,
4199 "and no faster than a heartbeat after that"
4200 );
4201 assert_eq!(unknown, 2);
4202 let replay = refusal(RefusalCode::Replay);
4203 assert_eq!(
4204 a.after_round(&replay, &mut failures, &mut unknown),
4205 a.backoff(3)
4206 );
4207 assert_eq!(
4208 a.after_round(&Err(RoundError::NoLink), &mut failures, &mut unknown),
4209 a.backoff(4)
4210 );
4211 assert_eq!(failures, 4);
4212 assert_eq!(a.after_round(&Ok(()), &mut failures, &mut unknown), beat);
4213 assert_eq!((failures, unknown), (0, 0));
4214 }
4215
4216 #[test]
4217 fn a_hello_links_both_ways_and_a_claim_crosses_the_link() {
4218 let net = Arc::new(InProcess::default());
4219 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4220 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4221 let key = a.dial(&["b:1".into()]).expect("B welcomes A");
4222 assert_eq!(key, b.runtime_key());
4223 assert_eq!(
4224 b.peers()[0].runtime,
4225 a.runtime_key(),
4226 "B holds A as its inbound peer"
4227 );
4228
4229 a.claim(
4230 &session("s1"),
4231 vec!["apps".into()],
4232 None,
4233 ClaimMode::Exclusive,
4234 Some("#184".into()),
4235 )
4236 .unwrap();
4237 a.sync_with(&key).unwrap();
4238 a.journal.beat_own();
4239 a.sync_with(&key).unwrap();
4240 let at_b = b.state();
4241 assert_eq!(at_b.claims.len(), 1, "B sees A's claim");
4242 let refused = b.claim(
4243 &session("s9"),
4244 vec!["apps/majordomus-cli".into()],
4245 None,
4246 ClaimMode::Exclusive,
4247 None,
4248 );
4249 assert!(
4250 matches!(refused, Err(CooperationError::Conflict(ref c)) if c.len() == 1),
4251 "{refused:?}"
4252 );
4253 assert_eq!(
4254 a.state().digest,
4255 b.state().digest,
4256 "one state, two runtimes"
4257 );
4258 }
4259
4260 #[test]
4261 fn a_withdrawn_key_loses_its_link_without_a_restart() {
4262 let net = Arc::new(InProcess::default());
4263 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4264 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4265 b.dial(&["a:1".into()]).expect("linked under tofu");
4266 assert_eq!(a.peers().len(), 1);
4267 // A's declaration now trusts nobody it does not list, and lists nobody
4268 let ended = a.retrust(TrustConfig::default());
4269 assert_eq!(ended, vec![b.runtime_key().to_string()], "B's link ended");
4270 // the link B already holds is gone on A's side: a sync over it is not served
4271 let synced = b.sync_with(a.runtime_key());
4272 assert!(
4273 matches!(synced, Err(RoundError::Refused(ref r)) if r.code == RefusalCode::UnknownLink),
4274 "{synced:?}"
4275 );
4276 // B's next sync meets a link A no longer knows, and its hello is refused
4277 let reply = b.dial(&["a:1".into()]);
4278 assert!(
4279 matches!(reply, Err(RoundError::Refused(ref r)) if r.code == RefusalCode::Untrusted),
4280 "{reply:?}"
4281 );
4282 // and a trust that admits B again lets it link again
4283 a.retrust(TrustConfig {
4284 policy: TrustPolicy::DenyUnknown,
4285 allow: vec![b.identity.public.public_key.clone()],
4286 });
4287 assert!(
4288 b.dial(&["a:1".into()]).is_ok(),
4289 "trusted again, linked again"
4290 );
4291 }
4292
4293 #[test]
4294 fn a_hello_addressed_to_one_runtime_is_refused_by_another() {
4295 let net = Arc::new(InProcess::default());
4296 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4297 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4298 let c = runtime(&net, "c:1", "root", TrustPolicy::Tofu);
4299 let addressed = |to: Option<String>| {
4300 let hello = Hello {
4301 proto_min: LINK_PROTOCOL_MIN,
4302 proto_max: LINK_PROTOCOL_MAX,
4303 card: a.card.clone(),
4304 nonce: fresh_token(),
4305 ts: crate::mesh::protocol::now(),
4306 to,
4307 };
4308 sign(
4309 &a.identity,
4310 Domain::Hello,
4311 serde_json::to_value(&hello).unwrap(),
4312 )
4313 };
4314 // a hello on its way to B, captured and replayed to C: refused, and C links nobody
4315 let to_b = addressed(Some(b.runtime_key().to_string()));
4316 let refusal = c.accept_hello(&to_b).refusal.expect("refused");
4317 assert_eq!(refusal.code, RefusalCode::Misaddressed);
4318 assert!(c.peers().is_empty(), "the replay changed no link of C");
4319 // the same hello is B's to accept
4320 assert!(
4321 b.accept_hello(&to_b).refusal.is_none(),
4322 "B accepts what was meant for it"
4323 );
4324 // a hello that names nobody is still accepted: a seed, a first contact, an older release
4325 assert!(c.accept_hello(&addressed(None)).refusal.is_none());
4326 // and the dialer addresses what it knows: a dial to B by key reaches B
4327 assert_eq!(
4328 a.dial_to(Some(b.runtime_key()), &["b:1".into()]).unwrap(),
4329 b.runtime_key()
4330 );
4331 let wrong = a.dial_to(Some(c.runtime_key()), &["b:1".into()]);
4332 assert!(
4333 matches!(wrong, Err(RoundError::Refused(ref r)) if r.code == RefusalCode::Misaddressed),
4334 "{wrong:?}"
4335 );
4336 }
4337
4338 #[test]
4339 fn isolation_version_trust_and_replay_are_typed_refusals() {
4340 let net = Arc::new(InProcess::default());
4341 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4342 let other_repo = runtime(&net, "x:1", "another-root", TrustPolicy::Tofu);
4343 let strict = runtime(&net, "s:1", "root", TrustPolicy::DenyUnknown);
4344
4345 let code = |r: Result<String, RoundError>| match r {
4346 Err(RoundError::Refused(refusal)) => refusal.code,
4347 other => panic!("expected a refusal, got {other:?}"),
4348 };
4349 assert_eq!(
4350 code(a.dial(&["x:1".into()])),
4351 RefusalCode::RepositoryMismatch
4352 );
4353 assert!(
4354 other_repo.peers().is_empty(),
4355 "no cooperative relationship formed"
4356 );
4357 assert_eq!(code(a.dial(&["s:1".into()])), RefusalCode::Untrusted);
4358 assert!(strict.peers().is_empty());
4359 assert_eq!(code(a.dial(&["a:1".into()])), RefusalCode::SelfLink);
4360
4361 // A hello from a future protocol range, properly signed.
4362 let future = Hello {
4363 proto_min: LINK_PROTOCOL_MAX + 1,
4364 proto_max: LINK_PROTOCOL_MAX + 5,
4365 card: a.card.clone(),
4366 nonce: fresh_token(),
4367 ts: crate::mesh::protocol::now(),
4368 to: None,
4369 };
4370 let signed = sign(
4371 &a.identity,
4372 Domain::Hello,
4373 serde_json::to_value(&future).unwrap(),
4374 );
4375 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4376 let reply = b.accept_hello(&signed);
4377 let refusal = reply.refusal.expect("refused");
4378 assert_eq!(refusal.code, RefusalCode::ProtocolUnsupported);
4379 assert_eq!(
4380 (refusal.proto_min, refusal.proto_max),
4381 (LINK_PROTOCOL_MIN, LINK_PROTOCOL_MAX)
4382 );
4383
4384 // A replayed hello. (A fresh nonce first: the refused future hello's nonce is
4385 // already spent, which is exactly the replay rule.)
4386 let hello = Hello {
4387 proto_min: 1,
4388 proto_max: 1,
4389 nonce: fresh_token(),
4390 ..future
4391 };
4392 let signed = sign(
4393 &a.identity,
4394 Domain::Hello,
4395 serde_json::to_value(&hello).unwrap(),
4396 );
4397 assert!(b.accept_hello(&signed).signed.is_some());
4398 assert_eq!(
4399 b.accept_hello(&signed).refusal.unwrap().code,
4400 RefusalCode::Replay
4401 );
4402
4403 // Garbage, a forged signature, and an oversized message.
4404 let garbage = Signed {
4405 body: serde_json::json!({"nope": true}),
4406 sig: "00".into(),
4407 };
4408 assert_eq!(
4409 b.accept_hello(&garbage).refusal.unwrap().code,
4410 RefusalCode::Malformed
4411 );
4412 let mut forged = sign(
4413 &a.identity,
4414 Domain::Hello,
4415 serde_json::to_value(&Hello {
4416 nonce: fresh_token(),
4417 ..hello.clone()
4418 })
4419 .unwrap(),
4420 );
4421 forged.body["card"]["name"] = serde_json::json!("impostor");
4422 assert_eq!(
4423 b.accept_hello(&forged).refusal.unwrap().code,
4424 RefusalCode::Signature
4425 );
4426 let huge = Signed {
4427 body: serde_json::json!({"x": "y".repeat(MAX_LINK_MESSAGE)}),
4428 sig: String::new(),
4429 };
4430 assert_eq!(
4431 b.accept_hello(&huge).refusal.unwrap().code,
4432 RefusalCode::Oversized
4433 );
4434 assert!(b.status().counters.refused_in.len() >= 4);
4435 }
4436
4437 #[test]
4438 fn a_sync_under_an_unknown_link_or_a_replayed_counter_is_refused() {
4439 let net = Arc::new(InProcess::default());
4440 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4441 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4442 let key = a.dial(&["b:1".into()]).unwrap();
4443 a.sync_with(&key).unwrap();
4444 let replay = {
4445 let table = a.table.lock().unwrap();
4446 let out = table.peers[&key].outbound.clone().unwrap();
4447 SyncRequest {
4448 link: out.link,
4449 counter: out.counter,
4450 instance: a.card.instance.clone(),
4451 marks: Marks::new(),
4452 events: vec![],
4453 }
4454 };
4455 let signed = sign(
4456 &a.identity,
4457 Domain::SyncRequest,
4458 serde_json::to_value(&replay).unwrap(),
4459 );
4460 assert_eq!(
4461 b.accept_sync(&signed).refusal.unwrap().code,
4462 RefusalCode::Replay
4463 );
4464 let unknown = SyncRequest {
4465 link: fresh_token(),
4466 counter: 99,
4467 ..replay
4468 };
4469 let signed = sign(
4470 &a.identity,
4471 Domain::SyncRequest,
4472 serde_json::to_value(&unknown).unwrap(),
4473 );
4474 assert_eq!(
4475 b.accept_sync(&signed).refusal.unwrap().code,
4476 RefusalCode::UnknownLink
4477 );
4478 }
4479
4480 #[test]
4481 fn three_runtimes_in_a_line_converge_without_loops_or_duplicates() {
4482 let net = Arc::new(InProcess::default());
4483 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4484 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4485 let c = runtime(&net, "c:1", "root", TrustPolicy::Tofu);
4486 let ab = a.dial(&["b:1".into()]).unwrap();
4487 let cb = c.dial(&["b:1".into()]).unwrap();
4488 a.claim(
4489 &session("sa"),
4490 vec!["apps".into()],
4491 None,
4492 ClaimMode::Exclusive,
4493 None,
4494 )
4495 .unwrap();
4496 c.claim(
4497 &session("sc"),
4498 vec!["docs".into()],
4499 None,
4500 ClaimMode::Exclusive,
4501 None,
4502 )
4503 .unwrap();
4504 for _ in 0..3 {
4505 for j in [&a, &b, &c] {
4506 j.journal.beat_own();
4507 }
4508 a.sync_with(&ab).unwrap();
4509 c.sync_with(&cb).unwrap();
4510 }
4511 let digests: Vec<String> = [&a, &b, &c].iter().map(|r| r.state().digest).collect();
4512 assert!(
4513 digests.windows(2).all(|w| w[0] == w[1]),
4514 "A, B and C converge: {digests:?}"
4515 );
4516 let state = c.state();
4517 assert_eq!(state.claims.len(), 2);
4518 assert_eq!(
4519 c.journal.events().len(),
4520 4,
4521 "two sessions and two claims, each once"
4522 );
4523 // More rounds carry nothing: replication stopped, no event loops back.
4524 let before = b.status().counters.events_served;
4525 a.sync_with(&ab).unwrap();
4526 c.sync_with(&cb).unwrap();
4527 assert_eq!(
4528 b.status().counters.events_served,
4529 before,
4530 "nothing left to serve"
4531 );
4532 assert_eq!(
4533 a.status().journal.duplicates + c.status().journal.duplicates,
4534 0
4535 );
4536 }
4537
4538 #[test]
4539 fn a_send_is_counted_before_the_peer_can_be_seen_holding_it() {
4540 // The three-runtime integration test reads the counters once every runtime holds
4541 // every event, and asserts that quiet rounds add nothing. On a slow runner that
4542 // reading fell between a peer ingesting a push and the push's answer reaching the
4543 // dialer, which counted the push only then: the send landed after the reading and
4544 // passed for an event travelling twice. The transport here stops in that window.
4545 let net = Arc::new(InProcess::default());
4546 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4547 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4548 let key = a.dial(&["b:1".into()]).unwrap();
4549 for (r, name, scope) in [(&a, "sa", "apps"), (&b, "sb", "docs")] {
4550 r.claim(
4551 &session(name),
4552 vec![scope.into()],
4553 None,
4554 ClaimMode::Exclusive,
4555 None,
4556 )
4557 .unwrap();
4558 }
4559 let seen: Arc<Mutex<Vec<(usize, u64, u64)>>> = Arc::default();
4560 let (dialer, peer, log) = (Arc::downgrade(&a), Arc::downgrade(&b), Arc::clone(&seen));
4561 *net.answered.lock().unwrap() = Some(Box::new(move || {
4562 let (Some(dialer), Some(peer)) = (dialer.upgrade(), peer.upgrade()) else {
4563 return;
4564 };
4565 let held = peer
4566 .journal
4567 .events()
4568 .iter()
4569 .filter(|e| &e.stream == dialer.journal.own_stream())
4570 .count();
4571 log.lock().unwrap().push((
4572 held,
4573 dialer.status().counters.events_sent,
4574 peer.status().counters.events_served,
4575 ));
4576 }));
4577 a.sync_with(&key).unwrap();
4578 a.sync_with(&key).unwrap();
4579 assert_eq!(
4580 *seen.lock().unwrap(),
4581 vec![(2, 2, 2), (2, 2, 2)],
4582 "while the answer is on its way the peer holds the dialer's session and claim, \
4583 and both sides already count what they sent; the quiet round adds nothing"
4584 );
4585 assert_eq!(
4586 a.status().counters.events_sent,
4587 2,
4588 "and the answer adds nothing"
4589 );
4590 assert_eq!(
4591 a.status().journal.duplicates + b.status().journal.duplicates,
4592 0
4593 );
4594 }
4595
4596 #[test]
4597 fn a_partition_expires_the_far_side_and_healing_reconciles_to_one_winner() {
4598 let net = Arc::new(InProcess::default());
4599 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4600 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4601 let key = a.dial(&["b:1".into()]).unwrap();
4602 a.journal.beat_own();
4603 b.journal.beat_own();
4604 a.sync_with(&key).unwrap();
4605
4606 // Partition: both sides claim the same scope, as each is allowed to.
4607 net.cut.lock().unwrap().push("b:1".into());
4608 assert!(matches!(a.sync_with(&key), Err(RoundError::Unreachable(_))));
4609 assert!(a.peers()[0].failures >= 1);
4610 a.claim(
4611 &session("s1"),
4612 vec!["apps".into()],
4613 None,
4614 ClaimMode::Exclusive,
4615 None,
4616 )
4617 .unwrap();
4618 std::thread::sleep(Duration::from_millis(3200));
4619 b.claim(
4620 &session("s2"),
4621 vec!["apps".into()],
4622 None,
4623 ClaimMode::Exclusive,
4624 None,
4625 )
4626 .expect("A's stream has expired at B, so nothing there refuses B");
4627
4628 // Heal: one round each way, and both name the same single winner.
4629 net.cut.lock().unwrap().clear();
4630 a.journal.beat_own();
4631 b.journal.beat_own();
4632 match a.sync_with(&key) {
4633 Ok(()) => {}
4634 Err(RoundError::Refused(r)) if r.code == RefusalCode::UnknownLink => {
4635 let key = a.dial(&["b:1".into()]).unwrap();
4636 a.sync_with(&key).unwrap();
4637 }
4638 Err(e) => panic!("{e}"),
4639 }
4640 let (sa, sb) = (a.state(), b.state());
4641 assert_eq!(sa.digest, sb.digest, "the partition healed into one state");
4642 let held = sa
4643 .claims
4644 .iter()
4645 .filter(|c| c.state == crate::mesh::state::ClaimState::Held)
4646 .count();
4647 let conflicted = sa
4648 .claims
4649 .iter()
4650 .filter(|c| matches!(c.state, crate::mesh::state::ClaimState::Conflicted(_)))
4651 .count();
4652 assert_eq!(
4653 (held, conflicted),
4654 (1, 1),
4655 "one winner, one named conflict — never two holders"
4656 );
4657 }
4658
4659 #[test]
4660 fn a_reviewer_must_carry_the_feature_and_stop_ends_every_thread() {
4661 let net = Arc::new(InProcess::default());
4662 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4663 let b = runtime(&net, "b:1", "root", TrustPolicy::Tofu);
4664 let key = a.dial(&["b:1".into()]).unwrap();
4665 a.table
4666 .lock()
4667 .unwrap()
4668 .peers
4669 .get_mut(&key)
4670 .unwrap()
4671 .card
4672 .features = vec!["claims".into()];
4673 let refused = a.request_review(
4674 "s1",
4675 "feature/x".into(),
4676 vec![],
4677 None,
4678 Some(key.clone()),
4679 None,
4680 );
4681 assert!(
4682 matches!(refused, Err(CooperationError::FeatureUnsupported(_))),
4683 "{refused:?}"
4684 );
4685 assert!(matches!(
4686 a.request_review(
4687 "s1",
4688 "feature/x".into(),
4689 vec![],
4690 None,
4691 Some("nobody".into()),
4692 None
4693 ),
4694 Err(CooperationError::NotFound(_))
4695 ));
4696 assert!(a
4697 .request_review("s1", "feature/x".into(), vec![], None, None, None)
4698 .is_ok());
4699
4700 a.start();
4701 b.start();
4702 std::thread::sleep(Duration::from_millis(300));
4703 assert!(a.status().counters.threads >= 1);
4704 a.stop();
4705 b.stop();
4706 let deadline = Instant::now() + Duration::from_secs(3);
4707 while (a.status().counters.threads > 0 || b.status().counters.threads > 0)
4708 && Instant::now() < deadline
4709 {
4710 std::thread::sleep(Duration::from_millis(50));
4711 }
4712 assert_eq!(
4713 a.status().counters.threads,
4714 0,
4715 "stop leaves no cooperation thread behind"
4716 );
4717 assert_eq!(
4718 a.accept_hello(&Signed {
4719 body: serde_json::json!({}),
4720 sig: String::new()
4721 })
4722 .refusal
4723 .unwrap()
4724 .code,
4725 RefusalCode::NotActive
4726 );
4727 }
4728
4729 #[test]
4730 fn the_board_is_projected_into_sessions_and_advisory_claims_and_withdrawn_with_it() {
4731 let net = Arc::new(InProcess::default());
4732 let board = Arc::new(crate::peers::PeerBoard::new());
4733 let identity = Arc::new(NodeIdentity::ephemeral().unwrap());
4734 let a = Cooperation::new(CooperationSetup {
4735 identity,
4736 runtime: "00000000000000a1".into(),
4737 repository: of_root_commits(&["root".into()]),
4738 endpoints: vec!["a:1".into()],
4739 version: "test".into(),
4740 config: CooperationConfig::default(),
4741 trust: TrustConfig::default(),
4742 journal_path: None,
4743 registry: Arc::new(MeshRegistry::new()),
4744 transport: Arc::clone(&net) as Arc<dyn LinkTransport>,
4745 board: Some(Arc::clone(&board)),
4746 checkout: CheckoutFacts::default(),
4747 })
4748 .unwrap();
4749 let p = board.attach(crate::peers::Transport::Http);
4750 board.announce(&p, "implement the mesh", vec!["apps/majordomus-cli".into()]);
4751 a.project_board();
4752 a.project_board();
4753 let state = a.state();
4754 assert_eq!(state.sessions.len(), 1);
4755 assert_eq!(
4756 state.sessions[0].info.intent.as_deref(),
4757 Some("implement the mesh")
4758 );
4759 assert_eq!(state.claims.len(), 1, "projecting twice claims once");
4760 assert_eq!(state.claims[0].mode, ClaimMode::Advisory);
4761
4762 board.announce(&p, "write the docs", vec!["docs".into()]);
4763 a.project_board();
4764 let state = a.state();
4765 let live: Vec<_> = state.claims.iter().filter(|c| c.state.is_live()).collect();
4766 assert_eq!(live.len(), 1);
4767 assert_eq!(live[0].scope, vec!["docs".to_string()]);
4768
4769 // an intent past the journal's bound is projected once, clipped, and then left alone:
4770 // compared unclipped it differed from the projection on every pass and reopened the
4771 // session each time (986 of 1000 journal events on 2026-10-10)
4772 let long = format!("relay the train {}", "x".repeat(600));
4773 board.announce(&p, &long, vec!["docs".into()]);
4774 a.project_board();
4775 let once = a.state();
4776 assert_eq!(
4777 once.sessions[0].info.intent.as_deref().map(str::len),
4778 Some(512),
4779 "clipped to the bound"
4780 );
4781 let events = a.journal().events().len();
4782 a.project_board();
4783 a.project_board();
4784 assert_eq!(
4785 a.journal().events().len(),
4786 events,
4787 "an unchanged long intent writes nothing"
4788 );
4789 assert_eq!(a.state().digest, once.digest);
4790
4791 board.detach(&p);
4792 a.project_board();
4793 let state = a.state();
4794 assert!(
4795 state.claims.iter().all(|c| !c.state.is_live()),
4796 "a departed session holds nothing"
4797 );
4798 }
4799
4800 #[test]
4801 fn a_release_is_the_holders_and_a_session_close_ends_its_claims() {
4802 let net = Arc::new(InProcess::default());
4803 let a = runtime(&net, "a:1", "root", TrustPolicy::Tofu);
4804 let w = a
4805 .claim(
4806 &session("s1"),
4807 vec!["apps".into()],
4808 None,
4809 ClaimMode::Exclusive,
4810 None,
4811 )
4812 .unwrap();
4813 assert!(matches!(
4814 a.release("someone-else/c-1"),
4815 Err(CooperationError::NotOwn(_))
4816 ));
4817 a.release(&w.key).unwrap();
4818 let w2 = a
4819 .claim(
4820 &session("s1"),
4821 vec!["docs".into()],
4822 None,
4823 ClaimMode::Exclusive,
4824 None,
4825 )
4826 .unwrap();
4827 a.close_session("s1").unwrap();
4828 let state = a.state();
4829 let second = state.claims.iter().find(|c| c.key == w2.key).unwrap();
4830 assert!(matches!(
4831 second.state,
4832 crate::mesh::state::ClaimState::Expired(_)
4833 ));
4834 }
4835
4836 /// A registry record of node `node` (as a 32-hex node id) for the dial selection: the
4837 /// runtime slot `runtime`, one endpoint, repository `root`, public key `key`.
4838 fn record(node: usize, runtime: &str, presence: Presence, key: &str) -> NodeRecord {
4839 NodeRecord {
4840 node_id: serde_json::from_value(serde_json::json!(format!("{node:032x}"))).unwrap(),
4841 runtime: runtime.into(),
4842 public_key: key.into(),
4843 display_name: format!("node-{node}"),
4844 instance_id: serde_json::from_value(serde_json::json!("0000000000000001")).unwrap(),
4845 trust: TrustState::Observed,
4846 presence,
4847 endpoints: vec![format!("{runtime}:1")],
4848 capabilities: vec![],
4849 repositories: vec!["root".into()],
4850 protocol_version: 2,
4851 version: "test".into(),
4852 sources: vec![],
4853 first_seen: String::new(),
4854 last_seen: String::new(),
4855 restarts: 0,
4856 }
4857 }
4858
4859 fn ids(targets: &[Target]) -> Vec<String> {
4860 targets.iter().map(|t| t.id.clone()).collect()
4861 }
4862
4863 /// The registry keeps a stopped runtime's record for its retention, fifteen minutes, and
4864 /// every runtime of a machine lists the registry in one order. Twenty stopped worktree
4865 /// servers whose keys sort before the five that run filled the per-node cap with the
4866 /// dead: no live runtime dialed a live sibling, and the machine's worktrees shared no
4867 /// claims until the stopped records expired.
4868 #[test]
4869 fn a_node_s_present_runtimes_are_dialed_before_the_stopped_ones() {
4870 let mut records: Vec<NodeRecord> = (0..20)
4871 .map(|i| record(1, &format!("{i:016}"), Presence::Absent, "trusted"))
4872 .collect();
4873 records.extend(
4874 (100..105).map(|i| record(1, &format!("{i:016}"), Presence::Present, "trusted")),
4875 );
4876 let key = |r: &NodeRecord| format!("{}-{}", r.node_id, r.runtime);
4877 let live: Vec<String> = records[20..].iter().map(key).collect();
4878 let own = live[0].clone();
4879
4880 let chosen = ids(&dial_targets(&records, "root", &own, |k| k == "trusted"));
4881
4882 assert_eq!(chosen.len(), MAX_TARGETS_PER_NODE, "the cap still holds");
4883 assert!(!chosen.contains(&own), "a runtime never dials itself");
4884 let siblings: Vec<&String> = live[1..].iter().filter(|k| chosen.contains(k)).collect();
4885 assert_eq!(
4886 siblings.len(),
4887 live.len() - 1,
4888 "every live sibling is dialed; chosen: {chosen:?}"
4889 );
4890 }
4891
4892 use proptest::strategy::{Just, Strategy};
4893
4894 proptest::proptest! {
4895 /// For any registry: of a node with present runtimes, min(cap, present) of them are
4896 /// dialed; of every node, min(cap, eligible); and the choice does not depend on the
4897 /// order the records arrive in.
4898 #[test]
4899 fn the_dial_selection_takes_present_runtimes_first_whatever_the_order(
4900 (mix, order) in proptest::collection::vec(
4901 (0usize..3, proptest::bool::ANY, 0usize..4),
4902 0..48,
4903 )
4904 .prop_flat_map(|mix| {
4905 let order: Vec<usize> = (0..mix.len()).collect();
4906 (Just(mix), Just(order).prop_shuffle())
4907 })
4908 ) {
4909 let records: Vec<NodeRecord> = mix
4910 .iter()
4911 .enumerate()
4912 .map(|(i, (node, present, kind))| {
4913 let presence = if *present { Presence::Present } else { Presence::Absent };
4914 let key = if *kind == 0 { "stranger" } else { "trusted" };
4915 record(*node, &format!("{i:016x}"), presence, key)
4916 })
4917 .collect();
4918 let shuffled: Vec<NodeRecord> = order.iter().map(|&i| records[i].clone()).collect();
4919 let own = format!("{:032x}-{:016x}", 0, 0);
4920 let trusted = |k: &str| k == "trusted";
4921
4922 let chosen = ids(&dial_targets(&records, "root", &own, trusted));
4923 proptest::prop_assert_eq!(
4924 &chosen,
4925 &ids(&dial_targets(&shuffled, "root", &own, trusted)),
4926 "the same records in another order choose the same targets"
4927 );
4928 for node in 0..3 {
4929 let prefix = format!("{node:032x}-");
4930 let eligible: Vec<(String, bool)> = records
4931 .iter()
4932 .filter(|r| r.node_id.as_str() == format!("{node:032x}"))
4933 .filter(|r| trusted(&r.public_key))
4934 .map(|r| {
4935 let key = format!("{}-{}", r.node_id, r.runtime);
4936 (key, r.presence == Presence::Present)
4937 })
4938 .filter(|(k, _)| *k != own)
4939 .collect();
4940 let present = eligible.iter().filter(|(_, p)| *p).count();
4941 let of_node: Vec<&String> =
4942 chosen.iter().filter(|k| k.starts_with(&prefix)).collect();
4943 let present_chosen = of_node
4944 .iter()
4945 .filter(|k| eligible.iter().any(|(e, p)| *p && e == **k))
4946 .count();
4947 let cap = MAX_TARGETS_PER_NODE;
4948 proptest::prop_assert_eq!(of_node.len(), eligible.len().min(cap));
4949 proptest::prop_assert_eq!(present_chosen, present.min(cap));
4950 }
4951 }
4952 }
4953}