1use std::io::{BufRead, BufReader, Write};
16use std::process::{Child, ChildStdin, Command, Stdio};
17use std::sync::Arc;
18use std::time::{Duration, Instant};
19
20use serde_json::{json, Value};
21
22use crate::capability::{CachePolicy, CapabilityError, Context};
23use crate::cli::{DiscoveryMode, RepoArgs};
24use crate::error::{Error, Result};
25use crate::http::server;
26use crate::http::Router;
27use crate::live::Live;
28use crate::mcp::bridge;
29
30use super::projection::{BenchmarkTarget, TargetKind, Transport, View};
31use super::results::{BenchmarkResult, CacheMode};
32use super::stats::Statistics;
33use super::system::SystemTarget;
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37pub struct Profile {
38 pub name: &'static str,
40 pub warmup: usize,
42 pub samples: usize,
44 pub cold_spawns: usize,
47 pub deferred: bool,
51}
52
53impl Profile {
54 pub const QUICK: Profile = Profile {
56 name: "quick",
57 warmup: 3,
58 samples: 20,
59 cold_spawns: 2,
60 deferred: false,
61 };
62 pub const FULL: Profile = Profile {
64 name: "full",
65 warmup: 10,
66 samples: 200,
67 cold_spawns: 10,
68 deferred: true,
69 };
70 pub const CI: Profile = Profile {
72 name: "ci",
73 warmup: 5,
74 samples: 50,
75 cold_spawns: 3,
76 deferred: false,
77 };
78
79 pub fn parse(name: &str) -> Option<Profile> {
81 match name {
82 "quick" => Some(Profile::QUICK),
83 "full" => Some(Profile::FULL),
84 "ci" => Some(Profile::CI),
85 _ => None,
86 }
87 }
88
89 pub fn admits(&self, target: &BenchmarkTarget) -> bool {
106 self.deferred || !target.is_deferred()
107 }
108}
109
110const PAGE_TIMEOUT: Duration = Duration::from_secs(120);
114const DEFERRED_TIMEOUT: Duration = Duration::from_secs(900);
115
116pub struct Runner {
118 ctx: Arc<Context>,
119 profile: Profile,
120 http: Option<server::Running>,
121 live_http: Option<(Arc<Live>, server::Running)>,
123 repo_args: Option<RepoArgs>,
126 mcp: Option<McpChild>,
127 executable: std::path::PathBuf,
129 repo_root: std::path::PathBuf,
130 share: Option<std::path::PathBuf>,
132 child_args: Vec<String>,
134}
135
136impl Runner {
137 pub fn new(ctx: Arc<Context>, profile: Profile, repo_root: &std::path::Path) -> Self {
139 Runner {
140 ctx,
141 profile,
142 http: None,
143 live_http: None,
144 repo_args: None,
145 mcp: None,
146 executable: std::env::current_exe().unwrap_or_else(|_| "majordomus".into()),
147 repo_root: repo_root.to_path_buf(),
148 share: None,
149 child_args: Vec::new(),
150 }
151 }
152
153 pub fn with_executable(mut self, path: std::path::PathBuf) -> Self {
155 self.executable = path;
156 self
157 }
158
159 pub fn with_share(mut self, share: std::path::PathBuf) -> Self {
161 self.share = Some(share);
162 self
163 }
164
165 pub fn with_repo_args(mut self, args: RepoArgs) -> Self {
191 self.repo_args = Some(args);
192 self
193 }
194
195 pub fn with_child_args(mut self, args: Vec<String>) -> Self {
198 self.child_args = args;
199 self
200 }
201
202 pub fn run(&mut self, target: &BenchmarkTarget) -> Result<Vec<BenchmarkResult>> {
204 match &target.kind {
205 TargetKind::Capability {
206 id,
207 transport,
208 input,
209 cache,
210 tool,
211 route,
212 ..
213 } => {
214 let modes: Vec<CacheMode> = if cache.is_enabled() {
215 vec![CacheMode::Cold, CacheMode::Warm]
216 } else {
217 vec![CacheMode::Uncached]
218 };
219 let mut out = Vec::new();
220 for mode in modes {
221 let (stats, invocations) = match transport {
222 Transport::Direct => self.direct(id, input, mode, *cache)?,
223 Transport::Http => {
224 let (method, path) = route.clone().ok_or_else(|| Error::Protocol {
225 reason: format!("{id}: HTTP target without a route"),
226 })?;
227 (self.http(&method, &path, input, mode)?, None)
228 }
229 Transport::Mcp => {
230 let tool = tool.clone().ok_or_else(|| Error::Protocol {
231 reason: format!("{id}: MCP target without a tool"),
232 })?;
233 (self.mcp_tool(&tool, input, mode)?, None)
234 }
235 };
236 out.push(BenchmarkResult {
237 key: target.key.clone(),
238 kind: target.kind.clone(),
239 cache_mode: mode,
240 stats,
241 handler_invocations: invocations,
242 });
243 }
244 Ok(out)
245 }
246 TargetKind::System { target: system } => Ok(vec![BenchmarkResult {
247 key: target.key.clone(),
248 kind: target.kind.clone(),
249 cache_mode: CacheMode::NotApplicable,
250 stats: self.system(*system)?,
251 handler_invocations: None,
252 }]),
253 TargetKind::Page {
254 route,
255 view,
256 deferred,
257 } => {
258 let mut out = Vec::new();
259 for mode in [CacheMode::Cold, CacheMode::Warm] {
260 out.push(BenchmarkResult {
261 key: target.key.clone(),
262 kind: target.kind.clone(),
263 cache_mode: mode,
264 stats: self.page(&target.key, route, *view, *deferred, mode)?,
265 handler_invocations: None,
266 });
267 }
268 Ok(out)
269 }
270 }
271 }
272
273 fn page(
279 &mut self,
280 key: &str,
281 route: &str,
282 view: View,
283 deferred: bool,
284 mode: CacheMode,
285 ) -> Result<Statistics> {
286 let (warmup, samples, timeout) = if deferred {
287 (
288 self.profile.warmup.min(1),
289 self.profile.cold_spawns.max(1),
290 DEFERRED_TIMEOUT,
291 )
292 } else {
293 (self.profile.warmup, self.profile.samples, PAGE_TIMEOUT)
294 };
295 let headers = [("Accept", "text/html")];
296 match view {
297 View::Pinned => {
298 let url = self.http_server()?;
299 let ctx = Arc::clone(&self.ctx);
300 let clear = move || {
301 if mode == CacheMode::Cold {
302 ctx.executor.clear();
303 }
304 };
305 time_requests(
306 &url, "GET", route, &headers, None, timeout, warmup, samples, clear,
307 )
308 }
309 View::Live => {
310 let url = self.live_server()?;
311 let live = self.live_http.as_ref().map(|(l, _)| Arc::clone(l));
312 let before = live.as_ref().map(|l| l.generation());
313 let clear = {
314 let live = live.clone();
315 move || {
316 if mode == CacheMode::Cold {
317 if let Some(live) = &live {
318 live.current().executor.clear();
321 }
322 }
323 }
324 };
325 let stats = time_requests(
326 &url, "GET", route, &headers, None, timeout, warmup, samples, clear,
327 )?;
328 if let (Some(live), Some(before)) = (live, before) {
329 let reloads = live.generation() - before;
330 if reloads > 0 {
331 tracing::warn!(
334 target = key,
335 reloads,
336 "the repository moved during the samples: the live view reloaded"
337 );
338 }
339 }
340 Ok(stats)
341 }
342 }
343 }
344
345 fn live_server(&mut self) -> Result<String> {
348 if self.live_http.is_none() {
349 let args = self.repo_args.clone().unwrap_or_else(|| RepoArgs {
350 repo: Some(self.repo_root.clone()),
351 discovery: if self.ctx.index.repository.discovery == "filesystem" {
352 DiscoveryMode::Filesystem
353 } else {
354 DiscoveryMode::Vcs
355 },
356 strict: false,
357 share: self.share.clone(),
358 });
359 let live = Arc::new(Live::watching(args, Arc::clone(&self.ctx)));
360 let bound = server::bind("127.0.0.1", 0)?;
361 let router = Router::new(Arc::clone(&live), crate::VERSION)
362 .with_cockpit(self.share.as_deref())
363 .with_ui(self.share.as_deref());
364 self.live_http = Some((live, bound.start(router)));
365 }
366 Ok(self
367 .live_http
368 .as_ref()
369 .map(|(_, r)| r.url())
370 .unwrap_or_default())
371 }
372
373 fn direct(
375 &mut self,
376 id: &str,
377 input: &Value,
378 mode: CacheMode,
379 _cache: CachePolicy,
380 ) -> Result<(Statistics, Option<u64>)> {
381 let ctx = Arc::clone(&self.ctx);
382 let ctx = match ctx.registry.get(id).map(|c| c.kind) {
384 Some(crate::capability::CapabilityKind::Command) => {
385 Arc::new(ctx.for_caller(ctx.peers.attach(crate::peers::Transport::Stdio)))
386 }
387 _ => ctx,
388 };
389 let fatal = |id: &str, e: CapabilityError| -> Option<Error> {
402 match e {
403 CapabilityError::Internal(reason) => Some(Error::Protocol {
404 reason: format!("{id}: {reason}"),
405 }),
406 _ => None,
407 }
408 };
409 let call = |ctx: &Context| -> Result<()> {
410 if mode == CacheMode::Cold {
411 ctx.executor.clear();
412 }
413 if let Err(e) = ctx.execute(id, input.clone()) {
414 if let Some(fatal) = fatal(id, e) {
415 return Err(fatal);
416 }
417 }
418 Ok(())
419 };
420 for _ in 0..self.profile.warmup {
421 call(&ctx)?;
422 }
423 let before = ctx.executor.handler_invocations();
424 let mut samples = Vec::with_capacity(self.profile.samples);
425 for _ in 0..self.profile.samples {
426 if mode == CacheMode::Cold {
427 ctx.executor.clear();
428 }
429 let t = Instant::now();
430 let answered = ctx.execute(id, input.clone());
431 samples.push(t.elapsed());
432 if let Err(e) = answered {
433 if let Some(fatal) = fatal(id, e) {
434 return Err(fatal);
435 }
436 }
437 }
438 let after = ctx.executor.handler_invocations();
439 Ok((Statistics::of(&samples), Some(after - before)))
440 }
441
442 fn http_server(&mut self) -> Result<String> {
443 if self.http.is_none() {
444 let bound = server::bind("127.0.0.1", 0)?;
445 let router = Router::new(Arc::clone(&self.ctx), crate::VERSION)
449 .with_cockpit(self.share.as_deref())
450 .with_ui(self.share.as_deref());
451 self.http = Some(bound.start(router));
452 }
453 Ok(self.http.as_ref().map(|r| r.url()).unwrap_or_default())
454 }
455
456 fn http(
458 &mut self,
459 method: &str,
460 path: &str,
461 input: &Value,
462 mode: CacheMode,
463 ) -> Result<Statistics> {
464 self.http_with(method, path, input, mode, &[])
465 }
466
467 fn http_with(
468 &mut self,
469 method: &str,
470 path: &str,
471 input: &Value,
472 mode: CacheMode,
473 headers: &[(&str, &str)],
474 ) -> Result<Statistics> {
475 let url = self.http_server()?;
476 let (target, body) = match method {
477 "GET" => (format!("{path}{}", query_string(input)), None),
478 _ => (path.to_string(), Some(input.to_string())),
479 };
480 let ctx = Arc::clone(&self.ctx);
481 let clear = move || {
482 if mode == CacheMode::Cold {
483 ctx.executor.clear();
484 }
485 };
486 time_requests(
487 &url,
488 method,
489 &target,
490 headers,
491 body.as_deref(),
492 Duration::from_secs(30),
493 self.profile.warmup,
494 self.profile.samples,
495 clear,
496 )
497 }
498
499 fn mcp_child(&mut self) -> Result<&mut McpChild> {
500 if self.mcp.is_none() {
501 let mut child = McpChild::spawn(
502 &self.executable,
503 &self.repo_root,
504 self.share.as_deref(),
505 &self.child_args,
506 )?;
507 child.initialize()?;
508 self.mcp = Some(child);
509 }
510 Ok(self.mcp.as_mut().expect("spawned"))
511 }
512
513 fn mcp_tool(&mut self, tool: &str, input: &Value, _mode: CacheMode) -> Result<Statistics> {
517 let profile = self.profile;
518 let child = self.mcp_child()?;
519 let frame = |id: u64| json!({ "jsonrpc": "2.0", "id": id, "method": "tools/call", "params": { "name": tool, "arguments": input } });
520 for _ in 0..profile.warmup {
521 let id = child.next_id();
522 child.round_trip(&frame(id))?;
523 }
524 let mut samples = Vec::with_capacity(profile.samples);
525 for _ in 0..profile.samples {
526 let f = frame(child.next_id());
527 let t = Instant::now();
528 let answer = child.round_trip(&f)?;
529 samples.push(t.elapsed());
530 if answer.get("error").is_some() {
531 return Err(Error::Protocol {
532 reason: format!("{tool}: {}", answer["error"]),
533 });
534 }
535 }
536 Ok(Statistics::of(&samples))
537 }
538
539 fn system(&mut self, target: SystemTarget) -> Result<Statistics> {
540 let profile = self.profile;
541 match target {
542 SystemTarget::McpProcessCold => {
543 let mut samples = Vec::new();
544 for _ in 0..profile.cold_spawns {
545 let t = Instant::now();
546 let mut child = McpChild::spawn(
547 &self.executable,
548 &self.repo_root,
549 self.share.as_deref(),
550 &self.child_args,
551 )?;
552 child.initialize()?;
553 let id = child.next_id();
554 child.round_trip(
555 &json!({ "jsonrpc": "2.0", "id": id, "method": "tools/list" }),
556 )?;
557 samples.push(t.elapsed());
558 child.close();
559 }
560 Ok(Statistics::of(&samples))
561 }
562 SystemTarget::McpInitialize
563 | SystemTarget::McpPing
564 | SystemTarget::McpToolsList
565 | SystemTarget::McpResourcesList
566 | SystemTarget::McpResourcesRead => {
567 let first_resource = self
568 .ctx
569 .index
570 .objects
571 .first()
572 .map(|o| o.uri.clone())
573 .unwrap_or_else(|| "majordomus://repository".into());
574 let child = self.mcp_child()?;
575 let frame = |id: u64| match target {
576 SystemTarget::McpInitialize => {
577 json!({ "jsonrpc": "2.0", "id": id, "method": "initialize", "params": { "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": { "name": "majordomus-bench", "version": crate::VERSION } } })
578 }
579 SystemTarget::McpPing => {
580 json!({ "jsonrpc": "2.0", "id": id, "method": "ping" })
581 }
582 SystemTarget::McpToolsList => {
583 json!({ "jsonrpc": "2.0", "id": id, "method": "tools/list" })
584 }
585 SystemTarget::McpResourcesList => {
586 json!({ "jsonrpc": "2.0", "id": id, "method": "resources/list" })
587 }
588 _ => {
589 json!({ "jsonrpc": "2.0", "id": id, "method": "resources/read", "params": { "uri": first_resource } })
590 }
591 };
592 for _ in 0..profile.warmup {
593 let id = child.next_id();
594 child.round_trip(&frame(id))?;
595 }
596 let mut samples = Vec::with_capacity(profile.samples);
597 for _ in 0..profile.samples {
598 let f = frame(child.next_id());
599 let t = Instant::now();
600 child.round_trip(&f)?;
601 samples.push(t.elapsed());
602 }
603 Ok(Statistics::of(&samples))
604 }
605 SystemTarget::HttpIndex
606 | SystemTarget::HttpHome
607 | SystemTarget::HttpOpenApi
608 | SystemTarget::HttpSwagger
609 | SystemTarget::HttpCockpitOverview
610 | SystemTarget::HttpCockpitCapabilities
611 | SystemTarget::HttpCockpitGraph
612 | SystemTarget::HttpIndexLive => {
613 let path = match target {
616 SystemTarget::HttpIndex
617 | SystemTarget::HttpHome
618 | SystemTarget::HttpIndexLive => "/",
619 SystemTarget::HttpOpenApi => crate::http::swagger::SPEC_PATH,
620 SystemTarget::HttpCockpitOverview => crate::cockpit::PREFIX,
621 SystemTarget::HttpCockpitCapabilities => "/cockpit/capabilities",
622 SystemTarget::HttpCockpitGraph => "/cockpit/graphs/registry",
623 _ => crate::http::swagger::SWAGGER_PATH,
624 };
625 let headers: &[(&str, &str)] = match target {
628 SystemTarget::HttpHome => &[("Accept", "text/html")],
629 _ => &[],
630 };
631 if target == SystemTarget::HttpIndexLive {
632 let url = self.live_server()?;
633 return time_requests(
634 &url,
635 "GET",
636 path,
637 headers,
638 None,
639 Duration::from_secs(30),
640 profile.warmup,
641 profile.samples,
642 || {},
643 );
644 }
645 self.http_with("GET", path, &json!({}), CacheMode::NotApplicable, headers)
646 }
647 }
648 }
649
650 pub fn finish(mut self) {
652 if let Some(child) = self.mcp.take() {
653 child.close();
654 }
655 if let Some(http) = self.http.take() {
656 http.stop();
657 }
658 if let Some((_, http)) = self.live_http.take() {
659 http.stop();
660 }
661 }
662}
663
664#[allow(clippy::too_many_arguments)]
668fn time_requests(
669 url: &str,
670 method: &str,
671 target: &str,
672 headers: &[(&str, &str)],
673 body: Option<&str>,
674 timeout: Duration,
675 warmup: usize,
676 samples: usize,
677 before: impl Fn(),
678) -> Result<Statistics> {
679 let once = || -> Result<Duration> {
680 before();
681 let t = Instant::now();
682 let reply = bridge::request(url, method, target, headers, body, timeout).map_err(|e| {
683 Error::Http {
684 reason: format!("{method} {target}: {e}"),
685 }
686 })?;
687 let elapsed = t.elapsed();
688 if reply.status >= 500 {
689 return Err(Error::Http {
690 reason: format!("{method} {target}: status {}", reply.status),
691 });
692 }
693 Ok(elapsed)
694 };
695 for _ in 0..warmup {
696 once()?;
697 }
698 let mut taken = Vec::with_capacity(samples);
699 for _ in 0..samples {
700 taken.push(once()?);
701 }
702 Ok(Statistics::of(&taken))
703}
704
705fn query_string(input: &Value) -> String {
707 let Some(map) = input.as_object() else {
708 return String::new();
709 };
710 let pairs: Vec<String> = map
711 .iter()
712 .filter(|(_, v)| !v.is_null())
713 .map(|(k, v)| {
714 let text = match v {
715 Value::String(s) => s.clone(),
716 other => other.to_string(),
717 };
718 format!("{}={}", percent_encode(k), percent_encode(&text))
719 })
720 .collect();
721 if pairs.is_empty() {
722 String::new()
723 } else {
724 format!("?{}", pairs.join("&"))
725 }
726}
727
728fn percent_encode(s: &str) -> String {
729 let mut out = String::new();
730 for b in s.bytes() {
731 match b {
732 b'A'..=b'Z'
733 | b'a'..=b'z'
734 | b'0'..=b'9'
735 | b'-'
736 | b'_'
737 | b'.'
738 | b'~'
739 | b'/'
740 | b':'
741 | b'@' => out.push(b as char),
742 _ => out.push_str(&format!("%{b:02X}")),
743 }
744 }
745 out
746}
747
748struct McpChild {
750 child: Child,
751 stdin: ChildStdin,
752 stdout: BufReader<std::process::ChildStdout>,
753 next: u64,
754}
755
756impl McpChild {
757 fn spawn(
758 executable: &std::path::Path,
759 repo_root: &std::path::Path,
760 share: Option<&std::path::Path>,
761 extra: &[String],
762 ) -> Result<Self> {
763 let mut command = Command::new(executable);
764 command
765 .args(["mcp", "--standalone"])
766 .args(extra)
767 .current_dir(repo_root)
768 .env("MAJORDOMUS_LOG", "warn");
769 if let Some(share) = share {
770 command.env("MAJORDOMUS_SHARE", share);
771 }
772 let mut child = command
773 .stdin(Stdio::piped())
774 .stdout(Stdio::piped())
775 .stderr(Stdio::null())
776 .spawn()
777 .map_err(|e| {
778 Error::Transport(std::io::Error::other(format!(
779 "spawn {}: {e}",
780 executable.display()
781 )))
782 })?;
783 let stdin = child.stdin.take().expect("piped");
784 let stdout = BufReader::new(child.stdout.take().expect("piped"));
785 Ok(McpChild {
786 child,
787 stdin,
788 stdout,
789 next: 0,
790 })
791 }
792
793 fn next_id(&mut self) -> u64 {
794 self.next += 1;
795 self.next
796 }
797
798 fn initialize(&mut self) -> Result<()> {
799 let id = self.next_id();
800 let answer = self.round_trip(&json!({ "jsonrpc": "2.0", "id": id, "method": "initialize", "params": { "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": { "name": "majordomus-bench", "version": crate::VERSION } } }))?;
801 if answer.get("result").is_none() {
802 return Err(Error::Protocol {
803 reason: format!("initialize failed: {answer}"),
804 });
805 }
806 writeln!(
807 self.stdin,
808 "{}",
809 json!({ "jsonrpc": "2.0", "method": "notifications/initialized" })
810 )
811 .map_err(Error::Transport)?;
812 Ok(())
813 }
814
815 fn round_trip(&mut self, frame: &Value) -> Result<Value> {
816 writeln!(self.stdin, "{frame}").map_err(Error::Transport)?;
817 self.stdin.flush().map_err(Error::Transport)?;
818 let mut line = String::new();
819 let n = self.stdout.read_line(&mut line).map_err(Error::Transport)?;
820 if n == 0 {
821 return Err(Error::Protocol {
822 reason: "the mcp child closed its stdout".into(),
823 });
824 }
825 serde_json::from_str(&line).map_err(|e| Error::Protocol {
826 reason: format!("not a frame: {e}: {line}"),
827 })
828 }
829
830 fn close(mut self) {
831 drop(self.stdin);
832 let _ = self.child.wait();
833 }
834}