From 1da6e0b191201b9afaab80ba243281942b17ad4a Mon Sep 17 00:00:00 2001 From: CN059 Date: Tue, 25 Aug 2026 01:00:18 +0800 Subject: [PATCH 1/3] test(control-plane): persist real microvm failures --- .../openoj-control-plane/tests/process_e2e.rs | 123 +++++++++++++++--- 1 file changed, 103 insertions(+), 20 deletions(-) diff --git a/apps/openoj-control-plane/tests/process_e2e.rs b/apps/openoj-control-plane/tests/process_e2e.rs index 035b7e7..eb942a0 100644 --- a/apps/openoj-control-plane/tests/process_e2e.rs +++ b/apps/openoj-control-plane/tests/process_e2e.rs @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use openoj_domain::{ExecutorKind, StageKind, Verdict}; +use openoj_domain::{ExecutorKind, StageKind, StageStatus, Verdict}; use openoj_protocol::decode_evaluation_result_domain; const VALID_REQUEST: &str = @@ -21,6 +21,12 @@ const ALGORITHM_C_OUTPUT_SHA256: &str = "084c799cd551dd1d8d5c5f9a5d593b2e931f5e36122ee5c793c1d08a19839cc0"; const ALGORITHM_C_OUTPUT_CONTENT_DIGEST: &str = "sha256:084c799cd551dd1d8d5c5f9a5d593b2e931f5e36122ee5c793c1d08a19839cc0"; +const ALGORITHM_C_COMPILE_ERROR_SOURCE: &[u8] = b"int main(void) { this is not C; }\n"; +const ALGORITHM_C_COMPILE_ERROR_SOURCE_SHA256: &str = + "45c508a05870369dd65fb951ee804127925b454fa0078ac6e37f84f5a1e3fa06"; +const ALGORITHM_C_TIMEOUT_SOURCE: &[u8] = b"int main(void) { for (;;) {} }\n"; +const ALGORITHM_C_TIMEOUT_SOURCE_SHA256: &str = + "84950edbf9514ebef845181f9f296bf2a71be3032f0d1d27b28b93fe1fd57f54"; #[test] fn postgres_uds_and_two_child_processes_reach_a_terminal_mock_result() -> Result<(), Box> @@ -75,6 +81,43 @@ fn postgres_uds_and_two_child_processes_reach_a_terminal_mock_result() -> Result #[test] fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result<(), Box> { + run_real_microvm_case(RealMicrovmCase { + source: ALGORITHM_C_SOURCE, + source_sha256: ALGORITHM_C_SOURCE_SHA256, + expected_verdict: Verdict::Accepted, + failed_stage: None, + }) +} + +#[test] +fn real_microvm_compile_error_is_persisted_and_reclaimed() -> Result<(), Box> { + run_real_microvm_case(RealMicrovmCase { + source: ALGORITHM_C_COMPILE_ERROR_SOURCE, + source_sha256: ALGORITHM_C_COMPILE_ERROR_SOURCE_SHA256, + expected_verdict: Verdict::CompileError, + failed_stage: Some(StageKind::Build), + }) +} + +#[test] +fn real_microvm_timeout_is_persisted_and_reclaimed() -> Result<(), Box> { + run_real_microvm_case(RealMicrovmCase { + source: ALGORITHM_C_TIMEOUT_SOURCE, + source_sha256: ALGORITHM_C_TIMEOUT_SOURCE_SHA256, + expected_verdict: Verdict::TimeLimitExceeded, + failed_stage: Some(StageKind::Run), + }) +} + +#[derive(Clone, Copy)] +struct RealMicrovmCase { + source: &'static [u8], + source_sha256: &'static str, + expected_verdict: Verdict, + failed_stage: Option, +} + +fn run_real_microvm_case(case: RealMicrovmCase) -> Result<(), Box> { let Some(images) = load_runtime_images()? else { return unavailable("algorithm-c runtime images are not provisioned"); }; @@ -94,10 +137,14 @@ fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result let api_socket = directory.path().join("firecracker.sock"); let vsock_socket = directory.path().join("vsock.sock"); let source_path = directory.path().join("main.c"); - fs::write(&source_path, ALGORITHM_C_SOURCE)?; + fs::write(&source_path, case.source)?; let request_path = directory.path().join("request.json"); - let evaluation_id = - unique_firecracker_request(&request_path, &format!("sha256:{}", images.runtime_sha256))?; + let evaluation_id = unique_firecracker_request( + &request_path, + &format!("sha256:{}", images.runtime_sha256), + case.source_sha256, + case.source.len(), + )?; let root = workspace_root()?; let mut control_plane = spawn_real_e2e_control_plane(&database_url, &socket_path)?; @@ -130,7 +177,7 @@ fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result let result = persisted_result(&database_url, &evaluation_id)?; assert_eq!( result.verdict(), - Verdict::Accepted, + case.expected_verdict, "persisted result: {result:?}" ); assert_eq!( @@ -145,18 +192,46 @@ fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result .map(openoj_domain::NodeId::as_str), Some("judge_node_fc_01") ); - let run = result - .stages() - .iter() - .find(|stage| stage.stage() == StageKind::Run) - .ok_or("run stage missing")?; - assert_eq!( - run.evidence() - .first() - .and_then(|evidence| evidence.artifact()) - .map(|artifact| artifact.digest().as_str()), - Some(ALGORITHM_C_OUTPUT_CONTENT_DIGEST) - ); + if let Some(failed_stage) = case.failed_stage { + let failed_index = result + .stages() + .iter() + .position(|stage| stage.stage() == failed_stage) + .ok_or("expected failed stage missing")?; + let failed = &result.stages()[failed_index]; + assert_eq!(failed.status(), StageStatus::Failed); + assert!( + !failed.diagnostics().is_empty(), + "failed stage must retain bounded diagnostics" + ); + assert!( + failed + .evidence() + .first() + .and_then(|evidence| evidence.artifact()) + .is_some(), + "failed stage must retain content-addressed evidence" + ); + assert!( + result.stages()[failed_index + 1..] + .iter() + .all(|stage| stage.status() == StageStatus::Skipped), + "stages after the terminal failure must be skipped" + ); + } else { + let run = result + .stages() + .iter() + .find(|stage| stage.stage() == StageKind::Run) + .ok_or("run stage missing")?; + assert_eq!( + run.evidence() + .first() + .and_then(|evidence| evidence.artifact()) + .map(|artifact| artifact.digest().as_str()), + Some(ALGORITHM_C_OUTPUT_CONTENT_DIGEST) + ); + } judge_node.stop(); control_plane.stop(); @@ -664,7 +739,12 @@ fn unique_request(path: &Path) -> Result> { Ok(format!("eval_{label}")) } -fn unique_firecracker_request(path: &Path, runtime_digest: &str) -> Result> { +fn unique_firecracker_request( + path: &Path, + runtime_digest: &str, + source_sha256: &str, + source_size: usize, +) -> Result> { let milliseconds = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(); let label = format!("fc{}_{}", std::process::id(), milliseconds); let runtime_hex = runtime_digest @@ -675,9 +755,12 @@ fn unique_firecracker_request(path: &Path, runtime_digest: &str) -> Result Date: Tue, 25 Aug 2026 01:01:53 +0800 Subject: [PATCH 2/3] test(control-plane): split microvm result assertions --- .../openoj-control-plane/tests/process_e2e.rs | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/apps/openoj-control-plane/tests/process_e2e.rs b/apps/openoj-control-plane/tests/process_e2e.rs index eb942a0..7b1ec84 100644 --- a/apps/openoj-control-plane/tests/process_e2e.rs +++ b/apps/openoj-control-plane/tests/process_e2e.rs @@ -175,6 +175,26 @@ fn run_real_microvm_case(case: RealMicrovmCase) -> Result<(), Box> { wait_for_any_terminal_status(&root, &database_url, &evaluation_id)?; let result = persisted_result(&database_url, &evaluation_id)?; + assert_real_microvm_result(&result, case)?; + + judge_node.stop(); + control_plane.stop(); + assert!( + !api_socket.exists(), + "Firecracker API socket was not reclaimed" + ); + assert!( + !vsock_socket.exists(), + "Firecracker vsock path was not reclaimed" + ); + fs::remove_file(socket_path)?; + Ok(()) +} + +fn assert_real_microvm_result( + result: &openoj_domain::EvaluationResult, + case: RealMicrovmCase, +) -> Result<(), Box> { assert_eq!( result.verdict(), case.expected_verdict, @@ -232,18 +252,6 @@ fn run_real_microvm_case(case: RealMicrovmCase) -> Result<(), Box> { Some(ALGORITHM_C_OUTPUT_CONTENT_DIGEST) ); } - - judge_node.stop(); - control_plane.stop(); - assert!( - !api_socket.exists(), - "Firecracker API socket was not reclaimed" - ); - assert!( - !vsock_socket.exists(), - "Firecracker vsock path was not reclaimed" - ); - fs::remove_file(socket_path)?; Ok(()) } From b8b53a73c4b6f9805d57c00a54d96b980a01e618 Mon Sep 17 00:00:00 2001 From: CN059 Date: Tue, 25 Aug 2026 01:04:54 +0800 Subject: [PATCH 3/3] docs(control-plane): record microvm failure evidence --- .../2026-08-25-algorithm-c-process-e2e.md | 4 + ...2026-08-25-algorithm-c-process-failures.md | 103 ++++++++++++++++++ docs/validation/README.md | 1 + ...026-08-25-algorithm-c-process-failures.txt | 61 +++++++++++ 4 files changed, 169 insertions(+) create mode 100644 docs/validation/2026-08-25-algorithm-c-process-failures.md create mode 100644 docs/validation/artifacts/2026-08-25-algorithm-c-process-failures.txt diff --git a/docs/validation/2026-08-25-algorithm-c-process-e2e.md b/docs/validation/2026-08-25-algorithm-c-process-e2e.md index 0592cd5..2762843 100644 --- a/docs/validation/2026-08-25-algorithm-c-process-e2e.md +++ b/docs/validation/2026-08-25-algorithm-c-process-e2e.md @@ -17,6 +17,10 @@ references: # algorithm-c 真实 microVM 进程闭环验证(2026-08-25) +> 后续证据:`2026-08-25-algorithm-c-process-failures.md` 已在真实 CLI/PostgreSQL/UDS/ +> judge-node/Firecracker 链路验证 `CompileError` 与 `TimeLimitExceeded` 的持久化、Evidence、 +> 后续 stage 跳过和资源回收。本记录保留原始 Accepted 闭环证据。 + ## 范围与结论 本记录验证 commit `c492288bcfc25485ca1bead9e9ff4ad31636ff4d` 的 development-only diff --git a/docs/validation/2026-08-25-algorithm-c-process-failures.md b/docs/validation/2026-08-25-algorithm-c-process-failures.md new file mode 100644 index 0000000..3b9b0a2 --- /dev/null +++ b/docs/validation/2026-08-25-algorithm-c-process-failures.md @@ -0,0 +1,103 @@ +--- +status: Validated +owners: OpenOJ control-plane, execution-plane, operations and security maintainers +last_reviewed: 2026-08-25 +applies_to: development-only algorithm-c real-microVM failure persistence at commit 752880c6f701256a86449a2e57be9592ba969703 +references: + - ../architecture/decisions/0005-firecracker-execution-plane.md + - ../operations/algorithm-c-development-runtime.md + - ../requirements/acceptance.md + - ../requirements/functional.md + - ../security/threat-model.md + - ../security/trust-boundaries.md + - 2026-08-25-algorithm-c-process-e2e.md + - 2026-08-25-algorithm-c-runtime-kvm.md +--- + +# algorithm-c 真实 microVM 失败终态验证(2026-08-25) + +## 范围与结论 + +本记录验证 commit `752880c6f701256a86449a2e57be9592ba969703`:固定编译错误与无限循环 +C Artifact 经过真实 CLI、PostgreSQL、control-plane UDS、judge-node、Firecracker/vsock 和 guest +agent 后,分别持久化为 `CompileError` 与 `TimeLimitExceeded`。失败 stage 保留有界诊断与内容 +寻址 Evidence,后续 stage 为 `Skipped`,VMM 与 socket 完成回收。 + +该证据扩展 `2026-08-25-algorithm-c-process-e2e.md` 的 Accepted 主路径,为 +`FR-EVAL-001`、`FR-JUDGE-002/003`、`FR-RESULT-001`、`ACC-P0-001/004/009/010/016` 与 +`RISK-EXEC-001` 提供 development-only 负路径的局部验证。它不实现正式对象存储、任意用户 +Artifact、取消/崩溃竞态、恶意资源耗尽矩阵或 production isolation。 + +## 固定 case 与断言 + +三个 case 使用同一参数化进程编排,每个 case 创建独立数据库、私有 UDS 和全新 microVM: + +| case | source SHA-256 | 持久化 Verdict | 终止 stage | +|---|---|---|---| +| Accepted | `c2329bffd207edd619d4ca7c7cdd872b76374e1a50421f705020671ca1f85556` | `Accepted` | Check 成功 | +| compile error | `45c508a05870369dd65fb951ee804127925b454fa0078ac6e37f84f5a1e3fa06` | `CompileError` | Build 失败 | +| infinite loop | `84950edbf9514ebef845181f9f296bf2a71be3032f0d1d27b28b93fe1fd57f54` | `TimeLimitExceeded` | Run 失败 | + +失败 case 共同断言:结果 provenance 为 Firecracker、`production_eligible=false`、node ID 为 +`judge_node_fc_01`;终止 stage 状态为 `Failed`,诊断非空且 Evidence 引用内容摘要 Artifact; +之后的 canonical stage 全部为 `Skipped`;judge-node 停止后 API/vsock 路径不存在。 + +## 环境与产物 + +- Arch Linux x86_64,kernel `7.1.8-zen1-3-zen`;AMD Ryzen 9 9950X3D,8 个可见 CPU, + 15 GiB 内存;VMware 暴露 AMD-V。 +- `/dev/kvm` 为 `crw-rw-rw- root:kvm`,KVM API `12`;Firecracker `1.16.1`;Rust/Cargo + `1.97.1`。 +- 一次性 PostgreSQL 18 Alpine 仅绑定 `127.0.0.1:55437`,数据位于 tmpfs,测试后已删除; + image digest 为 + `postgres@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2`。 + +| 产物 | SHA-256 | +|---|---| +| kernel | `882fa465c43ab7d92e31bd4167da3ad6a82cb9230f9b0016176df597c6014cef` | +| rootfs | `e9b2ca2f060161b01ccaafc20fd56e246def6b41011b4183d5c72f40b50f6f9c` | +| guest agent | `e775d7d0fe71b8980a58ad02b2a9b8ac1195d6b32fc745db659750e46373cae2` | +| runtime manifest | `c5ab7fb51144f39837e47799c2d7f1032f485b50a99337cb8fb84b41e3a12ea3` | +| SPDX SBOM | `aee7b475b81add5b58f497accdad5d4ef10a5ba8ad06e89007e2427e904ecddb` | + +本分支堆叠在真实进程闭环 PR 上,与 rootfs 可复现构建 PR 并列;测试从已校验 manifest 动态读取 +Runtime 摘要,不硬编码 rootfs。上表只描述本次实际验证产物,不替代可复现构建的独立证据。 + +## 命令与结果 + +以下命令通过: + +```text +OPENOJ_RUNTIME_OFFLINE=1 OPENOJ_RUNTIME_CACHE=/tmp/openoj-algorithm-c-cache bash infra/runtime-images/algorithm-c/provision.sh +(cd infra/runtime-images/algorithm-c/out && sha256sum --check --strict manifest.sha256) +OPENOJ_REQUIRE_KVM=1 OPENOJ_TEST_DATABASE_URL= cargo test -p openoj-control-plane --test process_e2e real_microvm_ --locked -- --nocapture --test-threads=1 +cargo fmt --all -- --check +cargo check --workspace --all-targets --locked +cargo clippy --workspace --all-targets --locked -- -D warnings +# 同时设置 DATABASE_URL、OPENOJ_TEST_DATABASE_URL 与 OPENOJ_REQUIRE_KVM=1: +cargo test --workspace --all-targets --locked +cargo deny --locked check advisories bans sources +python3 scripts/check-workspace.py +bash scripts/check-docs.sh +# 用带 PyYAML 的隔离解释器运行四个适用 Skill 的 quick_validate.py +``` + +聚焦的三个真实 microVM 进程 case 串行运行 25.18 秒,3/3 通过。完整 workspace 共 151 个测试 +通过、0 失败;其中 8 个 process E2E 在 13.16 秒通过,严格 runtime KVM smoke 在 1.87 秒通过, +17 个 PostgreSQL control-spine 测试通过。`cargo deny` 只有既有 duplicate-version 警告。 + +Clippy 首次拒绝 121 行的测试 helper;将持久化结果断言拆分后,在未添加 allow、未降低 lint 的 +情况下通过。测试后无 Firecracker 进程残留;临时 PostgreSQL 已删除,既有离线 cache 保留。 + +精简原始证据位于 `artifacts/2026-08-25-algorithm-c-process-failures.txt`,SHA-256 为 +`34def7013e0ffeaf1d80f0161a8905755f8aabc96ab548021019275d1b558b9a`。 + +## 安全影响与未验证项 + +本变更只扩展测试与验证证据,不修改协议、schema、migration、生产路径、网络、设备、凭证、 +argv 或信任边界。失败 Verdict 继续由宿主根据有界 guest stage output 形成,guest 不自行声明 +最终结果;production profile 继续 fail closed。 + +仍未验证:正式对象存储 Artifact 正文与短期授权、HTTP/API 与身份审计、取消/强杀/租约过期 +期间的真实 microVM 竞态、CPU/内存/process/disk/I/O/无限输出与并发 guest 宿主存活矩阵、 +production jailer/cgroup/namespace/seccomp/watchdog、第三方许可证人工复核及 release 性能。 diff --git a/docs/validation/README.md b/docs/validation/README.md index 58a6ef9..4aafff9 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -35,3 +35,4 @@ references: - `2026-08-23-algorithm-c-runtime-smoke.md`:`algorithm-c` 镜像供应、guest 上传摘要、严格 KVM smoke 与当前无 KVM 的未验证边界。 - `2026-08-25-algorithm-c-runtime-kvm.md`:修复 guest 文件系统启动后,在真实 KVM 上验证 C 编译运行、编译失败、运行超时与幂等回收。 - `2026-08-25-algorithm-c-process-e2e.md`:在真实 KVM 上验证 CLI、PostgreSQL、UDS、judge-node、Firecracker、宿主检查和持久化结果的固定 development Artifact 闭环。 +- `2026-08-25-algorithm-c-process-failures.md`:真实进程闭环持久化 CompileError 与 TimeLimitExceeded,保留失败证据并跳过后续阶段。 diff --git a/docs/validation/artifacts/2026-08-25-algorithm-c-process-failures.txt b/docs/validation/artifacts/2026-08-25-algorithm-c-process-failures.txt new file mode 100644 index 0000000..febd31b --- /dev/null +++ b/docs/validation/artifacts/2026-08-25-algorithm-c-process-failures.txt @@ -0,0 +1,61 @@ +OpenOJ algorithm-c real-microVM process failure validation +Date: 2026-08-25 Asia/Shanghai +Implementation commit: 752880c6f701256a86449a2e57be9592ba969703 + +Environment +Linux 7.1.8-zen1-3-zen x86_64 +AMD Ryzen 9 9950X3D, 8 visible CPUs, 15 GiB memory, VMware AMD-V +/dev/kvm crw-rw-rw- root:kvm; KVM API version 12 +Firecracker v1.16.1 +Rust/Cargo 1.97.1 +PostgreSQL 18-alpine image: +postgres@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2 + +Fixtures +accepted source: c2329bffd207edd619d4ca7c7cdd872b76374e1a50421f705020671ca1f85556 +accepted output: 084c799cd551dd1d8d5c5f9a5d593b2e931f5e36122ee5c793c1d08a19839cc0 +compile-error source: 45c508a05870369dd65fb951ee804127925b454fa0078ac6e37f84f5a1e3fa06 +timeout source: 84950edbf9514ebef845181f9f296bf2a71be3032f0d1d27b28b93fe1fd57f54 + +Runtime artifacts +kernel: 882fa465c43ab7d92e31bd4167da3ad6a82cb9230f9b0016176df597c6014cef +rootfs: e9b2ca2f060161b01ccaafc20fd56e246def6b41011b4183d5c72f40b50f6f9c +guest agent: e775d7d0fe71b8980a58ad02b2a9b8ac1195d6b32fc745db659750e46373cae2 +manifest: c5ab7fb51144f39837e47799c2d7f1032f485b50a99337cb8fb84b41e3a12ea3 +SPDX SBOM: aee7b475b81add5b58f497accdad5d4ef10a5ba8ad06e89007e2427e904ecddb +manifest checksum file: 7f52e4517fd503418599fe538a8f9b0ec839eaad650450c60728458b62d612ed + +Focused strict process test +$ OPENOJ_REQUIRE_KVM=1 OPENOJ_TEST_DATABASE_URL= \ + cargo test -p openoj-control-plane --test process_e2e real_microvm_ \ + --locked -- --nocapture --test-threads=1 +postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result: PASS +real_microvm_compile_error_is_persisted_and_reclaimed: PASS +real_microvm_timeout_is_persisted_and_reclaimed: PASS +3 passed; 0 failed; 25.18 s + +Assertions for both failure cases +- persisted verdict matches CompileError or TimeLimitExceeded +- executor provenance is Firecracker, production_eligible is false, node ID is judge_node_fc_01 +- the terminal build/run stage is Failed and contains diagnostics plus content-addressed evidence +- all stages after the terminal rejection are Skipped +- Firecracker API and vsock paths are reclaimed + +Workspace gates +cargo fmt --all -- --check: PASS +cargo check --workspace --all-targets --locked: PASS +cargo clippy --workspace --all-targets --locked -- -D warnings: PASS +DATABASE_URL= OPENOJ_TEST_DATABASE_URL= OPENOJ_REQUIRE_KVM=1 \ + cargo test --workspace --all-targets --locked: PASS (151 tests) +process_e2e: PASS (8 tests, 13.16 s) +strict algorithm-c KVM runtime smoke: PASS (1 test, 1.87 s) +PostgreSQL control-spine: PASS (17 tests) +cargo deny --locked check advisories bans sources: PASS (existing duplicate-version warnings only) +python3 scripts/check-workspace.py: PASS + +Clippy first rejected a 121-line helper under clippy::too_many_lines. Result assertions were +extracted into a separate helper; Clippy then passed without an allow attribute or weaker lint. + +Cleanup +The task-local PostgreSQL container was removed. No Firecracker process remained. +The pre-existing shared offline runtime cache was retained.