Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 123 additions & 32 deletions apps/openoj-control-plane/tests/process_e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use openoj_domain::{ExecutorKind, StageKind, Verdict};
use openoj_domain::{ExecutorKind, StageKind, StageStatus, Verdict};
use openoj_protocol::decode_evaluation_result_domain;

const VALID_REQUEST: &str =
Expand All @@ -21,6 +21,12 @@ const ALGORITHM_C_OUTPUT_SHA256: &str =
"084c799cd551dd1d8d5c5f9a5d593b2e931f5e36122ee5c793c1d08a19839cc0";
const ALGORITHM_C_OUTPUT_CONTENT_DIGEST: &str =
"sha256:084c799cd551dd1d8d5c5f9a5d593b2e931f5e36122ee5c793c1d08a19839cc0";
const ALGORITHM_C_COMPILE_ERROR_SOURCE: &[u8] = b"int main(void) { this is not C; }\n";
const ALGORITHM_C_COMPILE_ERROR_SOURCE_SHA256: &str =
"45c508a05870369dd65fb951ee804127925b454fa0078ac6e37f84f5a1e3fa06";
const ALGORITHM_C_TIMEOUT_SOURCE: &[u8] = b"int main(void) { for (;;) {} }\n";
const ALGORITHM_C_TIMEOUT_SOURCE_SHA256: &str =
"84950edbf9514ebef845181f9f296bf2a71be3032f0d1d27b28b93fe1fd57f54";

#[test]
fn postgres_uds_and_two_child_processes_reach_a_terminal_mock_result() -> Result<(), Box<dyn Error>>
Expand Down Expand Up @@ -75,6 +81,43 @@ fn postgres_uds_and_two_child_processes_reach_a_terminal_mock_result() -> Result
#[test]
fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result<(), Box<dyn Error>>
{
run_real_microvm_case(RealMicrovmCase {
source: ALGORITHM_C_SOURCE,
source_sha256: ALGORITHM_C_SOURCE_SHA256,
expected_verdict: Verdict::Accepted,
failed_stage: None,
})
}

#[test]
fn real_microvm_compile_error_is_persisted_and_reclaimed() -> Result<(), Box<dyn Error>> {
run_real_microvm_case(RealMicrovmCase {
source: ALGORITHM_C_COMPILE_ERROR_SOURCE,
source_sha256: ALGORITHM_C_COMPILE_ERROR_SOURCE_SHA256,
expected_verdict: Verdict::CompileError,
failed_stage: Some(StageKind::Build),
})
}

#[test]
fn real_microvm_timeout_is_persisted_and_reclaimed() -> Result<(), Box<dyn Error>> {
run_real_microvm_case(RealMicrovmCase {
source: ALGORITHM_C_TIMEOUT_SOURCE,
source_sha256: ALGORITHM_C_TIMEOUT_SOURCE_SHA256,
expected_verdict: Verdict::TimeLimitExceeded,
failed_stage: Some(StageKind::Run),
})
}

#[derive(Clone, Copy)]
struct RealMicrovmCase {
source: &'static [u8],
source_sha256: &'static str,
expected_verdict: Verdict,
failed_stage: Option<StageKind>,
}

fn run_real_microvm_case(case: RealMicrovmCase) -> Result<(), Box<dyn Error>> {
let Some(images) = load_runtime_images()? else {
return unavailable("algorithm-c runtime images are not provisioned");
};
Expand All @@ -94,10 +137,14 @@ fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result
let api_socket = directory.path().join("firecracker.sock");
let vsock_socket = directory.path().join("vsock.sock");
let source_path = directory.path().join("main.c");
fs::write(&source_path, ALGORITHM_C_SOURCE)?;
fs::write(&source_path, case.source)?;
let request_path = directory.path().join("request.json");
let evaluation_id =
unique_firecracker_request(&request_path, &format!("sha256:{}", images.runtime_sha256))?;
let evaluation_id = unique_firecracker_request(
&request_path,
&format!("sha256:{}", images.runtime_sha256),
case.source_sha256,
case.source.len(),
)?;
let root = workspace_root()?;

let mut control_plane = spawn_real_e2e_control_plane(&database_url, &socket_path)?;
Expand Down Expand Up @@ -128,9 +175,29 @@ fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result

wait_for_any_terminal_status(&root, &database_url, &evaluation_id)?;
let result = persisted_result(&database_url, &evaluation_id)?;
assert_real_microvm_result(&result, case)?;

judge_node.stop();
control_plane.stop();
assert!(
!api_socket.exists(),
"Firecracker API socket was not reclaimed"
);
assert!(
!vsock_socket.exists(),
"Firecracker vsock path was not reclaimed"
);
fs::remove_file(socket_path)?;
Ok(())
}

fn assert_real_microvm_result(
result: &openoj_domain::EvaluationResult,
case: RealMicrovmCase,
) -> Result<(), Box<dyn Error>> {
assert_eq!(
result.verdict(),
Verdict::Accepted,
case.expected_verdict,
"persisted result: {result:?}"
);
assert_eq!(
Expand All @@ -145,30 +212,46 @@ fn postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result() -> Result
.map(openoj_domain::NodeId::as_str),
Some("judge_node_fc_01")
);
let run = result
.stages()
.iter()
.find(|stage| stage.stage() == StageKind::Run)
.ok_or("run stage missing")?;
assert_eq!(
run.evidence()
.first()
.and_then(|evidence| evidence.artifact())
.map(|artifact| artifact.digest().as_str()),
Some(ALGORITHM_C_OUTPUT_CONTENT_DIGEST)
);

judge_node.stop();
control_plane.stop();
assert!(
!api_socket.exists(),
"Firecracker API socket was not reclaimed"
);
assert!(
!vsock_socket.exists(),
"Firecracker vsock path was not reclaimed"
);
fs::remove_file(socket_path)?;
if let Some(failed_stage) = case.failed_stage {
let failed_index = result
.stages()
.iter()
.position(|stage| stage.stage() == failed_stage)
.ok_or("expected failed stage missing")?;
let failed = &result.stages()[failed_index];
assert_eq!(failed.status(), StageStatus::Failed);
assert!(
!failed.diagnostics().is_empty(),
"failed stage must retain bounded diagnostics"
);
assert!(
failed
.evidence()
.first()
.and_then(|evidence| evidence.artifact())
.is_some(),
"failed stage must retain content-addressed evidence"
);
assert!(
result.stages()[failed_index + 1..]
.iter()
.all(|stage| stage.status() == StageStatus::Skipped),
"stages after the terminal failure must be skipped"
);
} else {
let run = result
.stages()
.iter()
.find(|stage| stage.stage() == StageKind::Run)
.ok_or("run stage missing")?;
assert_eq!(
run.evidence()
.first()
.and_then(|evidence| evidence.artifact())
.map(|artifact| artifact.digest().as_str()),
Some(ALGORITHM_C_OUTPUT_CONTENT_DIGEST)
);
}
Ok(())
}

Expand Down Expand Up @@ -664,7 +747,12 @@ fn unique_request(path: &Path) -> Result<String, Box<dyn Error>> {
Ok(format!("eval_{label}"))
}

fn unique_firecracker_request(path: &Path, runtime_digest: &str) -> Result<String, Box<dyn Error>> {
fn unique_firecracker_request(
path: &Path,
runtime_digest: &str,
source_sha256: &str,
source_size: usize,
) -> Result<String, Box<dyn Error>> {
let milliseconds = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis();
let label = format!("fc{}_{}", std::process::id(), milliseconds);
let runtime_hex = runtime_digest
Expand All @@ -675,9 +763,12 @@ fn unique_firecracker_request(path: &Path, runtime_digest: &str) -> Result<Strin
.replace("idem_01", &format!("idem_{label}"))
.replace("eval_01", &format!("eval_{label}"))
.replace("attempt_01", &format!("attempt_{label}"))
.replace(&"2".repeat(64), ALGORITHM_C_SOURCE_SHA256)
.replace(&"2".repeat(64), source_sha256)
.replace("text/x-c++src", "text/x-csrc")
.replace("\"size_bytes\": 128", "\"size_bytes\": 60")
.replace(
"\"size_bytes\": 128",
&format!("\"size_bytes\": {source_size}"),
)
.replace("runtime_cpp_01", "runtime_algorithm_c_v0alpha1")
.replace(&"3".repeat(64), runtime_hex);
fs::write(path, request)?;
Expand Down
4 changes: 4 additions & 0 deletions docs/validation/2026-08-25-algorithm-c-process-e2e.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ references:

# algorithm-c 真实 microVM 进程闭环验证(2026-08-25)

> 后续证据:`2026-08-25-algorithm-c-process-failures.md` 已在真实 CLI/PostgreSQL/UDS/
> judge-node/Firecracker 链路验证 `CompileError` 与 `TimeLimitExceeded` 的持久化、Evidence、
> 后续 stage 跳过和资源回收。本记录保留原始 Accepted 闭环证据。

## 范围与结论

本记录验证 commit `c492288bcfc25485ca1bead9e9ff4ad31636ff4d` 的 development-only
Expand Down
103 changes: 103 additions & 0 deletions docs/validation/2026-08-25-algorithm-c-process-failures.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
---
status: Validated
owners: OpenOJ control-plane, execution-plane, operations and security maintainers
last_reviewed: 2026-08-25
applies_to: development-only algorithm-c real-microVM failure persistence at commit 752880c6f701256a86449a2e57be9592ba969703
references:
- ../architecture/decisions/0005-firecracker-execution-plane.md
- ../operations/algorithm-c-development-runtime.md
- ../requirements/acceptance.md
- ../requirements/functional.md
- ../security/threat-model.md
- ../security/trust-boundaries.md
- 2026-08-25-algorithm-c-process-e2e.md
- 2026-08-25-algorithm-c-runtime-kvm.md
---

# algorithm-c 真实 microVM 失败终态验证(2026-08-25)

## 范围与结论

本记录验证 commit `752880c6f701256a86449a2e57be9592ba969703`:固定编译错误与无限循环
C Artifact 经过真实 CLI、PostgreSQL、control-plane UDS、judge-node、Firecracker/vsock 和 guest
agent 后,分别持久化为 `CompileError` 与 `TimeLimitExceeded`。失败 stage 保留有界诊断与内容
寻址 Evidence,后续 stage 为 `Skipped`,VMM 与 socket 完成回收。

该证据扩展 `2026-08-25-algorithm-c-process-e2e.md` 的 Accepted 主路径,为
`FR-EVAL-001`、`FR-JUDGE-002/003`、`FR-RESULT-001`、`ACC-P0-001/004/009/010/016` 与
`RISK-EXEC-001` 提供 development-only 负路径的局部验证。它不实现正式对象存储、任意用户
Artifact、取消/崩溃竞态、恶意资源耗尽矩阵或 production isolation。

## 固定 case 与断言

三个 case 使用同一参数化进程编排,每个 case 创建独立数据库、私有 UDS 和全新 microVM:

| case | source SHA-256 | 持久化 Verdict | 终止 stage |
|---|---|---|---|
| Accepted | `c2329bffd207edd619d4ca7c7cdd872b76374e1a50421f705020671ca1f85556` | `Accepted` | Check 成功 |
| compile error | `45c508a05870369dd65fb951ee804127925b454fa0078ac6e37f84f5a1e3fa06` | `CompileError` | Build 失败 |
| infinite loop | `84950edbf9514ebef845181f9f296bf2a71be3032f0d1d27b28b93fe1fd57f54` | `TimeLimitExceeded` | Run 失败 |

失败 case 共同断言:结果 provenance 为 Firecracker、`production_eligible=false`、node ID 为
`judge_node_fc_01`;终止 stage 状态为 `Failed`,诊断非空且 Evidence 引用内容摘要 Artifact;
之后的 canonical stage 全部为 `Skipped`;judge-node 停止后 API/vsock 路径不存在。

## 环境与产物

- Arch Linux x86_64,kernel `7.1.8-zen1-3-zen`;AMD Ryzen 9 9950X3D,8 个可见 CPU,
15 GiB 内存;VMware 暴露 AMD-V。
- `/dev/kvm` 为 `crw-rw-rw- root:kvm`,KVM API `12`;Firecracker `1.16.1`;Rust/Cargo
`1.97.1`。
- 一次性 PostgreSQL 18 Alpine 仅绑定 `127.0.0.1:55437`,数据位于 tmpfs,测试后已删除;
image digest 为
`postgres@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2`。

| 产物 | SHA-256 |
|---|---|
| kernel | `882fa465c43ab7d92e31bd4167da3ad6a82cb9230f9b0016176df597c6014cef` |
| rootfs | `e9b2ca2f060161b01ccaafc20fd56e246def6b41011b4183d5c72f40b50f6f9c` |
| guest agent | `e775d7d0fe71b8980a58ad02b2a9b8ac1195d6b32fc745db659750e46373cae2` |
| runtime manifest | `c5ab7fb51144f39837e47799c2d7f1032f485b50a99337cb8fb84b41e3a12ea3` |
| SPDX SBOM | `aee7b475b81add5b58f497accdad5d4ef10a5ba8ad06e89007e2427e904ecddb` |

本分支堆叠在真实进程闭环 PR 上,与 rootfs 可复现构建 PR 并列;测试从已校验 manifest 动态读取
Runtime 摘要,不硬编码 rootfs。上表只描述本次实际验证产物,不替代可复现构建的独立证据。

## 命令与结果

以下命令通过:

```text
OPENOJ_RUNTIME_OFFLINE=1 OPENOJ_RUNTIME_CACHE=/tmp/openoj-algorithm-c-cache bash infra/runtime-images/algorithm-c/provision.sh
(cd infra/runtime-images/algorithm-c/out && sha256sum --check --strict manifest.sha256)
OPENOJ_REQUIRE_KVM=1 OPENOJ_TEST_DATABASE_URL=<PostgreSQL 18> cargo test -p openoj-control-plane --test process_e2e real_microvm_ --locked -- --nocapture --test-threads=1
cargo fmt --all -- --check
cargo check --workspace --all-targets --locked
cargo clippy --workspace --all-targets --locked -- -D warnings
# 同时设置 DATABASE_URL、OPENOJ_TEST_DATABASE_URL 与 OPENOJ_REQUIRE_KVM=1:
cargo test --workspace --all-targets --locked
cargo deny --locked check advisories bans sources
python3 scripts/check-workspace.py
bash scripts/check-docs.sh
# 用带 PyYAML 的隔离解释器运行四个适用 Skill 的 quick_validate.py
```

聚焦的三个真实 microVM 进程 case 串行运行 25.18 秒,3/3 通过。完整 workspace 共 151 个测试
通过、0 失败;其中 8 个 process E2E 在 13.16 秒通过,严格 runtime KVM smoke 在 1.87 秒通过,
17 个 PostgreSQL control-spine 测试通过。`cargo deny` 只有既有 duplicate-version 警告。

Clippy 首次拒绝 121 行的测试 helper;将持久化结果断言拆分后,在未添加 allow、未降低 lint 的
情况下通过。测试后无 Firecracker 进程残留;临时 PostgreSQL 已删除,既有离线 cache 保留。

精简原始证据位于 `artifacts/2026-08-25-algorithm-c-process-failures.txt`,SHA-256 为
`34def7013e0ffeaf1d80f0161a8905755f8aabc96ab548021019275d1b558b9a`。

## 安全影响与未验证项

本变更只扩展测试与验证证据,不修改协议、schema、migration、生产路径、网络、设备、凭证、
argv 或信任边界。失败 Verdict 继续由宿主根据有界 guest stage output 形成,guest 不自行声明
最终结果;production profile 继续 fail closed。

仍未验证:正式对象存储 Artifact 正文与短期授权、HTTP/API 与身份审计、取消/强杀/租约过期
期间的真实 microVM 竞态、CPU/内存/process/disk/I/O/无限输出与并发 guest 宿主存活矩阵、
production jailer/cgroup/namespace/seccomp/watchdog、第三方许可证人工复核及 release 性能。
1 change: 1 addition & 0 deletions docs/validation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@ references:
- `2026-08-23-algorithm-c-runtime-smoke.md`:`algorithm-c` 镜像供应、guest 上传摘要、严格 KVM smoke 与当前无 KVM 的未验证边界。
- `2026-08-25-algorithm-c-runtime-kvm.md`:修复 guest 文件系统启动后,在真实 KVM 上验证 C 编译运行、编译失败、运行超时与幂等回收。
- `2026-08-25-algorithm-c-process-e2e.md`:在真实 KVM 上验证 CLI、PostgreSQL、UDS、judge-node、Firecracker、宿主检查和持久化结果的固定 development Artifact 闭环。
- `2026-08-25-algorithm-c-process-failures.md`:真实进程闭环持久化 CompileError 与 TimeLimitExceeded,保留失败证据并跳过后续阶段。
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
OpenOJ algorithm-c real-microVM process failure validation
Date: 2026-08-25 Asia/Shanghai
Implementation commit: 752880c6f701256a86449a2e57be9592ba969703

Environment
Linux 7.1.8-zen1-3-zen x86_64
AMD Ryzen 9 9950X3D, 8 visible CPUs, 15 GiB memory, VMware AMD-V
/dev/kvm crw-rw-rw- root:kvm; KVM API version 12
Firecracker v1.16.1
Rust/Cargo 1.97.1
PostgreSQL 18-alpine image:
postgres@sha256:d3e1620b530c944afa6e887d22eb899824da68e19c52024bf98f5220c88a65b2

Fixtures
accepted source: c2329bffd207edd619d4ca7c7cdd872b76374e1a50421f705020671ca1f85556
accepted output: 084c799cd551dd1d8d5c5f9a5d593b2e931f5e36122ee5c793c1d08a19839cc0
compile-error source: 45c508a05870369dd65fb951ee804127925b454fa0078ac6e37f84f5a1e3fa06
timeout source: 84950edbf9514ebef845181f9f296bf2a71be3032f0d1d27b28b93fe1fd57f54

Runtime artifacts
kernel: 882fa465c43ab7d92e31bd4167da3ad6a82cb9230f9b0016176df597c6014cef
rootfs: e9b2ca2f060161b01ccaafc20fd56e246def6b41011b4183d5c72f40b50f6f9c
guest agent: e775d7d0fe71b8980a58ad02b2a9b8ac1195d6b32fc745db659750e46373cae2
manifest: c5ab7fb51144f39837e47799c2d7f1032f485b50a99337cb8fb84b41e3a12ea3
SPDX SBOM: aee7b475b81add5b58f497accdad5d4ef10a5ba8ad06e89007e2427e904ecddb
manifest checksum file: 7f52e4517fd503418599fe538a8f9b0ec839eaad650450c60728458b62d612ed

Focused strict process test
$ OPENOJ_REQUIRE_KVM=1 OPENOJ_TEST_DATABASE_URL=<isolated-postgresql-18> \
cargo test -p openoj-control-plane --test process_e2e real_microvm_ \
--locked -- --nocapture --test-threads=1
postgres_uds_and_judge_node_reach_a_persisted_real_microvm_result: PASS
real_microvm_compile_error_is_persisted_and_reclaimed: PASS
real_microvm_timeout_is_persisted_and_reclaimed: PASS
3 passed; 0 failed; 25.18 s

Assertions for both failure cases
- persisted verdict matches CompileError or TimeLimitExceeded
- executor provenance is Firecracker, production_eligible is false, node ID is judge_node_fc_01
- the terminal build/run stage is Failed and contains diagnostics plus content-addressed evidence
- all stages after the terminal rejection are Skipped
- Firecracker API and vsock paths are reclaimed

Workspace gates
cargo fmt --all -- --check: PASS
cargo check --workspace --all-targets --locked: PASS
cargo clippy --workspace --all-targets --locked -- -D warnings: PASS
DATABASE_URL=<isolated-postgresql-18> OPENOJ_TEST_DATABASE_URL=<same> OPENOJ_REQUIRE_KVM=1 \
cargo test --workspace --all-targets --locked: PASS (151 tests)
process_e2e: PASS (8 tests, 13.16 s)
strict algorithm-c KVM runtime smoke: PASS (1 test, 1.87 s)
PostgreSQL control-spine: PASS (17 tests)
cargo deny --locked check advisories bans sources: PASS (existing duplicate-version warnings only)
python3 scripts/check-workspace.py: PASS

Clippy first rejected a 121-line helper under clippy::too_many_lines. Result assertions were
extracted into a separate helper; Clippy then passed without an allow attribute or weaker lint.

Cleanup
The task-local PostgreSQL container was removed. No Firecracker process remained.
The pre-existing shared offline runtime cache was retained.