diff --git a/tools/xtask-llm-benchmark/src/bench/mod.rs b/tools/xtask-llm-benchmark/src/bench/mod.rs index 6b1ebb1cf98..16f7da94cef 100644 --- a/tools/xtask-llm-benchmark/src/bench/mod.rs +++ b/tools/xtask-llm-benchmark/src/bench/mod.rs @@ -6,7 +6,7 @@ mod templates; pub mod types; pub(crate) mod utils; -pub use publishers::{DotnetPublisher, Publisher, SpacetimeRustPublisher, TypeScriptPublisher}; +pub use publishers::{DotnetPublisher, PublishedDatabase, Publisher, SpacetimeRustPublisher, TypeScriptPublisher}; pub use runner::TaskRunner; pub use types::{RunOutcome, TaskPaths}; pub use utils::bench_route_concurrency; diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index b7fb74c6936..626450ed19a 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -205,8 +205,63 @@ fn strip_ansi_codes(s: &str) -> Cow<'_, str> { /* Shared */ /* -------------------------------------------------------------------------- */ +pub struct PublishedDatabase { + host_url: String, + db: String, + // Keep the publishing CLI session alive so deletion uses the database + // owner's credentials without sharing mutable CLI state across routes. + cli_root: CliRootDir, + deleted: bool, +} + +impl PublishedDatabase { + fn new(host_url: &str, db: String, cli_root: CliRootDir) -> Self { + Self { + host_url: host_url.to_owned(), + db, + cli_root, + deleted: false, + } + } + + fn delete_inner(&self) -> Result<()> { + let mut cmd = spacetime_cmd(&self.cli_root); + cmd.arg("delete") + .arg("-y") + .arg("--no-config") + .arg("--server") + .arg(&self.host_url) + .arg(&self.db); + run(&mut cmd, "spacetime delete").with_context(|| { + format!( + "failed to clean up benchmark database `{}` on {}", + self.db, self.host_url + ) + }) + } + + pub fn delete(mut self) -> Result<()> { + self.delete_inner()?; + self.deleted = true; + Ok(()) + } +} + +impl Drop for PublishedDatabase { + fn drop(&mut self) { + if !self.deleted + && let Err(error) = self.delete_inner() + { + eprintln!( + "[cleanup] failed to delete benchmark database `{}` during fallback cleanup: {error:#}", + self.db + ); + } + } +} + pub trait Publisher: Send + Sync { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()>; + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result; } /// Check if the process was killed by a signal (e.g., SIGSEGV = 11) @@ -342,7 +397,7 @@ impl DotnetPublisher { } impl Publisher for DotnetPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -370,7 +425,7 @@ impl Publisher for DotnetPublisher { Self::configure_dotnet_env(&mut pubcmd); run(&mut pubcmd, "spacetime publish (csharp)")?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } /* -------------------------------------------------------------------------- */ @@ -390,7 +445,7 @@ impl SpacetimeRustPublisher { } impl Publisher for SpacetimeRustPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -416,7 +471,7 @@ impl Publisher for SpacetimeRustPublisher { "spacetime publish", )?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } @@ -437,7 +492,7 @@ impl TypeScriptPublisher { } impl Publisher for TypeScriptPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -539,6 +594,6 @@ impl Publisher for TypeScriptPublisher { } run(&mut publish_cmd, "spacetime publish (typescript)")?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 2536b5e5fe1..565cd071861 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -1,16 +1,14 @@ use anyhow::{anyhow, bail, Context, Result}; use chrono::Utc; -use futures::{stream, StreamExt}; +use futures::{stream, StreamExt, TryStreamExt}; use serde_json::json; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::OnceLock; -use std::time::Instant; -use tokio::sync::Mutex; +use std::time::{Duration, Instant}; use tokio::task; -use crate::bench::publishers::{DotnetPublisher, SpacetimeRustPublisher, TypeScriptPublisher}; +use crate::bench::publishers::{DotnetPublisher, PublishedDatabase, SpacetimeRustPublisher, TypeScriptPublisher}; use crate::bench::templates::materialize_project; use crate::bench::types::{BenchRunContext, PublishParams, RunContext, RunOneError}; pub(crate) use crate::bench::types::{RunOutcome, TaskPaths}; @@ -30,56 +28,43 @@ pub struct TaskRunner { pub ts_publisher: TypeScriptPublisher, } -static BUILT_KEYS: OnceLock>> = OnceLock::new(); - struct PendingRetry { task: TaskPaths, error: anyhow::Error, } -fn build_key(lang: Lang, selectors: Option<&[String]>) -> String { - let v = match selectors { - Some(s) if !s.is_empty() => { - let mut t = s.to_vec(); - t.sort(); // stable key independent of order - t +fn task_result_or_infrastructure_error( + task: TaskPaths, + result: Result, + started: chrono::DateTime, +) -> Result<(TaskPaths, Result)> { + match result { + Err(RunOneError::Infrastructure { msg, llm_output }) => { + let output_note = llm_output + .as_deref() + .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) + .unwrap_or_default(); + bail!("benchmark infrastructure failure: {msg}{output_note}"); } - _ => vec!["ALL".to_string()], - }; - let joined = v.join(","); - format!("{lang:?}:{joined}") + other => Ok(( + task, + other.map(|mut outcome| { + outcome.started_at.get_or_insert(started); + outcome + }), + )), + } } -/// Build goldens **once per (lang, selector-set)** in this process. -/// If selectors is None/empty, that means "ALL tasks". +/// Build the golden databases and return handles that keep their owning CLI +/// sessions alive until scoring has finished. pub async fn ensure_goldens_built_once( host: Option, bench_root: &Path, lang: Lang, selectors: Option<&[String]>, -) -> Result<()> { - let key = build_key(lang, selectors); - let set = BUILT_KEYS.get_or_init(|| Mutex::new(HashSet::new())); - { - let set = set.lock(); - if set.await.contains(&key) { - return Ok(()); - } - } - // single-flight for this key - let set_guard = set.lock().await; - if set_guard.contains(&key) { - return Ok(()); - } - - // IMPORTANT: pass selectors through so we only build needed goldens - build_goldens_only_for_lang(host, bench_root, lang, selectors).await?; - - // mark as built - drop(set_guard); - let mut set = BUILT_KEYS.get().unwrap().lock().await; - set.insert(key); - Ok(()) +) -> Result> { + build_goldens_only_for_lang(host, bench_root, lang, selectors).await } async fn publish_rust_async( @@ -87,16 +72,57 @@ async fn publish_rust_async( host_url: String, wdir: PathBuf, db: String, -) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await? +} +async fn publish_cs_async( + publisher: DotnetPublisher, + host_url: String, + wdir: PathBuf, + db: String, +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await? +} +async fn publish_ts_async( + publisher: TypeScriptPublisher, + host_url: String, + wdir: PathBuf, + db: String, +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await? +} + +async fn delete_database_async(database: PublishedDatabase) -> Result<()> { + task::spawn_blocking(move || database.delete()).await??; Ok(()) } -async fn publish_cs_async(publisher: DotnetPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + +pub async fn delete_databases(databases: Vec) -> Result<()> { + stream::iter(databases.into_iter().map(delete_database_async)) + .buffer_unordered(8) + .try_collect::>() + .await?; Ok(()) } -async fn publish_ts_async(publisher: TypeScriptPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + +async fn ensure_server_healthy(host: Option<&str>) -> Result<()> { + let Some(host) = host else { + return Ok(()); + }; + let ping_url = format!("{}/v1/ping", host.trim_end_matches('/')); + let response = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build()? + .get(&ping_url) + .send() + .await + .with_context(|| format!("benchmark server at {host} is unavailable; refusing to upload results"))?; + if !response.status().is_success() { + bail!( + "benchmark server health check at {ping_url} returned {}; refusing to upload results", + response.status() + ); + } Ok(()) } @@ -123,7 +149,7 @@ impl TaskRunner { golden_src_text: &str, golden_db: String, host: Option, - ) -> Result<()> { + ) -> Result { self.publish( PublishParams { lang, @@ -139,11 +165,11 @@ impl TaskRunner { .await } - async fn publish_llm(&self, params: PublishParams<'_>) -> Result<()> { + async fn publish_llm(&self, params: PublishParams<'_>) -> Result { self.publish(params, "llm").await } - async fn publish(&self, params: PublishParams<'_>, phase: &str) -> Result<()> { + async fn publish(&self, params: PublishParams<'_>, phase: &str) -> Result { let lang_name = match params.lang { Lang::Rust => "rust", Lang::CSharp => "csharp", @@ -164,13 +190,13 @@ impl TaskRunner { )?; let host_url = params.host.unwrap_or_else(|| "local".to_owned()); - match params.lang { + let database = match params.lang { Lang::Rust => publish_rust_async(self.rust_publisher, host_url, wdir, params.db_name).await?, Lang::CSharp => publish_cs_async(self.cs_publisher, host_url, wdir, params.db_name).await?, Lang::TypeScript => publish_ts_async(self.ts_publisher, host_url, wdir, params.db_name).await?, - } + }; - Ok(()) + Ok(database) } pub async fn run_one(&self, task: &TaskPaths, cfg: &RunContext<'_>) -> Result { @@ -267,7 +293,7 @@ impl TaskRunner { print_llm_output(&cfg.route.display_name, &task_id, &llm_output); } - let publish_error: Option = self + let publish_result = self .publish_llm(PublishParams { lang: cfg.lang, category: &category, @@ -277,15 +303,26 @@ impl TaskRunner { db_name: llm_db.clone(), host: cfg.host.clone(), }) - .await - .err() - .map(|e| { + .await; + let (published_database, publish_error) = match publish_result { + Ok(database) => (Some(database), None), + Err(error) => { eprintln!( - "⚠️ publish failed for {}/{}/{}: {e:#}", + "⚠️ publish failed for {}/{}/{}: {error:#}", category, task_id, cfg.route.display_name ); - format!("{:#}", e) + (None, Some(format!("{error:#}"))) + } + }; + + if publish_error.is_some() + && let Err(error) = ensure_server_healthy(cfg.host.as_deref()).await + { + return Err(RunOneError::Infrastructure { + msg: format!("publish failed and the benchmark server is unavailable: {error:#}"), + llm_output: Some(llm_output), }); + } let mut passed = 0usize; let mut partial_sum = 0f32; @@ -319,6 +356,15 @@ impl TaskRunner { ); } + if let Some(database) = published_database + && let Err(error) = delete_database_async(database).await + { + return Err(RunOneError::Infrastructure { + msg: format!("database cleanup failed for `{llm_db}`: {error:#}"), + llm_output: Some(llm_output), + }); + } + let score_pct = if total_tasks == 0 { 0.0 } else { @@ -381,7 +427,7 @@ fn partition_results( lang_name: &str, route: &ModelRoute, hash: &str, -) -> (Vec, Vec) { +) -> Result<(Vec, Vec)> { let mut good = Vec::new(); let mut retry = Vec::new(); @@ -400,6 +446,13 @@ fn partition_results( Some(llm_output), )); } + Err(RunOneError::Infrastructure { msg, llm_output }) => { + let output_note = llm_output + .as_deref() + .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) + .unwrap_or_default(); + bail!("benchmark infrastructure failure: {msg}{output_note}"); + } Err(RunOneError::Other(e)) => { // No output at all — provider error, retry eprintln!("\u{26a0}\u{fe0f} provider error, will retry: {e:?}"); @@ -408,7 +461,7 @@ fn partition_results( } } - (good, retry) + Ok((good, retry)) } fn pending_retries_to_outcomes( @@ -532,21 +585,15 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); + let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; // Retry provider-error tasks until all pass or none make progress const MAX_RETRY_ROUNDS: usize = 3; @@ -586,21 +633,15 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu llm, host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; if new_outcomes.is_empty() && !still_failing.is_empty() { // No progress — provider is likely down. Keep the failures and stop retrying. @@ -632,6 +673,10 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu println!("[runner] completed batch: {} results", outcomes.len()); + // A dead local server turns later publish failures into misleading zeroes. + // Refuse to analyze or upload any batch that cannot pass a final health check. + ensure_server_healthy(cfg.host.as_deref()).await?; + if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { Ok(text) => text, @@ -728,21 +773,15 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> host: cfg.host.clone(), }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); + let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; // Retry provider-error tasks until all pass or none make progress let dropped; @@ -785,21 +824,15 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> llm, host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; if new_outcomes.is_empty() && !still_failing.is_empty() { eprintln!( @@ -827,6 +860,8 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> ); } + ensure_server_healthy(cfg.host.as_deref()).await?; + if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { Ok(text) => text, @@ -886,7 +921,7 @@ pub async fn build_goldens_only_for_lang( bench_root: &Path, lang: Lang, selectors: Option<&[String]>, -) -> Result<()> { +) -> Result> { let tasks = if let Some(sels) = selectors { let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; let all = discover_tasks(bench_root)?; @@ -922,7 +957,7 @@ pub async fn build_goldens_only_for_lang( _ => bench_concurrency(), }; - stream::iter(tasks.into_iter().map(|task| { + let databases = stream::iter(tasks.into_iter().map(|task| { let runner = &runner; let host_clone = host.clone(); async move { @@ -943,7 +978,7 @@ pub async fn build_goldens_only_for_lang( .collect::>>()?; println!("✓ [{}] goldens build/publish: complete", lang_name); - Ok(()) + Ok(databases) } fn discover_tasks(benchmarks_root: &Path) -> Result> { diff --git a/tools/xtask-llm-benchmark/src/bench/templates.rs b/tools/xtask-llm-benchmark/src/bench/templates.rs index 35176de8200..3d86a4fdb29 100644 --- a/tools/xtask-llm-benchmark/src/bench/templates.rs +++ b/tools/xtask-llm-benchmark/src/bench/templates.rs @@ -235,6 +235,7 @@ fn write_csharp_nuget_config(root: &Path) -> Result<()> { + @@ -244,6 +245,10 @@ fn write_csharp_nuget_config(root: &Path) -> Result<()> { + + + + diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index e54df0d4902..16a8336a61d 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -134,6 +134,8 @@ pub struct RouteRun { pub enum RunOneError { #[error("{msg}")] WithOutput { msg: String, llm_output: String }, + #[error("{msg}")] + Infrastructure { msg: String, llm_output: Option }, #[error(transparent)] Other(#[from] anyhow::Error), } diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 6ec030a49e8..db9148fa337 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -13,7 +13,8 @@ use tokio::runtime::Runtime; use xtask_llm_benchmark::api::ApiClient; use xtask_llm_benchmark::bench::bench_route_concurrency; use xtask_llm_benchmark::bench::runner::{ - build_goldens_only_for_lang, ensure_goldens_built_once, run_selected_or_all_for_model_async_for_lang, + build_goldens_only_for_lang, delete_databases, ensure_goldens_built_once, + run_selected_or_all_for_model_async_for_lang, }; use xtask_llm_benchmark::bench::types::{BenchRunContext, RouteRun, RunConfig, RunOutcome}; use xtask_llm_benchmark::context::constants::ALL_MODES; @@ -279,7 +280,7 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { eprintln!("[warn] failed to upload task catalog: {e}"); } - let RuntimeInit { runtime, guard } = initialize_runtime(config.hash_only)?; + let RuntimeInit { runtime, mut guard } = initialize_runtime(config.hash_only)?; config.host = guard.as_ref().map(|g| g.host_url.clone()); @@ -295,48 +296,88 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { if config.goldens_only { let rt = runtime.as_ref().expect("runtime required for --goldens-only"); - rt.block_on(build_goldens_only_for_lang( + let result = rt.block_on(build_goldens_only_for_lang( config.host.clone(), &bench_root, config.lang, selectors_ref, - ))?; + )); + let databases = match result { + Ok(databases) => databases, + Err(error) => { + report_server_status(guard.as_mut()); + return Err(error); + } + }; + if let Err(error) = rt.block_on(delete_databases(databases)) { + report_server_status(guard.as_mut()); + return Err(error); + } println!("[{}] goldens-only build complete", config.lang.as_str()); return Ok(()); } - let llm_provider = if !config.goldens_only && !config.hash_only { + // Goldens are live parity fixtures: scorers describe them, call their + // reducers, and compare their data with generated modules. Keep them + // published through every model and mode, then delete them at teardown. + let (llm_provider, golden_databases) = if !config.goldens_only && !config.hash_only { let rt = runtime.as_ref().expect("failed to initialize runtime for goldens"); - rt.block_on(ensure_goldens_built_once( + let result = rt.block_on(ensure_goldens_built_once( config.host.clone(), &bench_root, config.lang, selectors_ref, - ))?; + )); + let databases = match result { + Ok(databases) => databases, + Err(error) => { + report_server_status(guard.as_mut()); + return Err(error); + } + }; let provider = make_provider_from_env()?; let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); let routes = filter_routes(&config); preflight_llm_routes(rt, provider.as_ref(), &routes, &modes)?; - Some(provider) + (Some(provider), databases) } else { - None + (None, Vec::new()) }; let mut all_outcomes: Vec = Vec::new(); for mode in modes { - let outcomes = run_mode_benchmarks( + let result = run_mode_benchmarks( &mode, config.lang, &config, &bench_root, runtime.as_ref(), llm_provider.as_ref(), - )?; + ); + let outcomes = match result { + Ok(outcomes) => outcomes, + Err(error) => { + if let Some(rt) = runtime.as_ref() + && let Err(cleanup_error) = rt.block_on(delete_databases(golden_databases)) + { + eprintln!("[cleanup] failed to delete golden databases after benchmark failure: {cleanup_error:#}"); + } + report_server_status(guard.as_mut()); + return Err(error); + } + }; all_outcomes.extend(outcomes); } + if let Some(rt) = runtime.as_ref() + && let Err(error) = rt.block_on(delete_databases(golden_databases)) + { + report_server_status(guard.as_mut()); + return Err(error); + } + // Write local run log on --dry-run so results aren't lost if dry_run && !all_outcomes.is_empty() { let runs_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("runs"); @@ -358,6 +399,17 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { Ok(()) } +fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { + let Some(guard) = guard else { + return; + }; + match guard.child.try_wait() { + Ok(Some(status)) => eprintln!("[server] local SpacetimeDB exited unexpectedly: {status}"), + Ok(None) => eprintln!("[server] local SpacetimeDB is still running after benchmark failure"), + Err(error) => eprintln!("[server] failed to read local SpacetimeDB exit status: {error}"), + } +} + /* ------------------------------ analyze ------------------------------ */ fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { diff --git a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj index f286932badd..d401d2c7575 100644 --- a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj +++ b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj @@ -1,7 +1,7 @@ - net8.0 + net8.0;net10.0 wasi-wasm enable enable