From 756dad1c55210bb6f5732276b83d865b167f0818 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 07:22:39 +0000 Subject: [PATCH 1/8] deps: move zip to 9.x for RUSTSEC-2026-0367/0368/0369 cargo-audit fails on every PR: zip 2.4.2 in elastos/Cargo.lock and in capsules/object-provider/Cargo.lock (through its elastos-server path dependency) has three advisories against ZipArchive::extract and the stream extract API. The runtime calls neither; it reads each entry itself. zip 9.0.4 is the patched release. - elastos-server uses zip 9.0.4, still without default features. 9.x has no separate flate2 feature; deflate-flate2 now enables the dependency. - Both lockfiles change only for zip: nix 0.31.3 and typed-path come in, derive_arbitrary goes. `cargo update -p zip` also moved unrelated range dependencies (windows-sys, getrandom); those lines stay as they were, and `cargo metadata --locked` accepts both files. - ZipFile::name now returns a Result. The ZIP readers report an error as "invalid ZIP archive entry". - Whole-archive extract keeps its rule for entry names. zip 9 enclosed_name turns "/a.txt" into "a.txt" and folds "x/../a.txt" into "a.txt", so an entry that extract refuses today would be written. The runtime now checks the name with the rule zip 2.x applied: no NUL byte, no root or drive prefix, no climb above the archive root. test_library_provider_extract_zip_refuses_absolute_dotdot_and_symlink_entries covers absolute, "..", inner "..", backslash and symlink entries. It passes on zip 2.4.2 and with this change, and fails on the Absolute.zip case when extract uses zip 9 enclosed_name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- capsules/object-provider/Cargo.lock | 50 ++--- elastos/Cargo.lock | 50 ++--- elastos/crates/elastos-server/Cargo.toml | 2 +- .../src/api/gateway_tests/library.rs | 199 +++++++++++++++++- elastos/crates/elastos-server/src/library.rs | 44 +++- 5 files changed, 290 insertions(+), 55 deletions(-) diff --git a/capsules/object-provider/Cargo.lock b/capsules/object-provider/Cargo.lock index d5114091d..914d1dc39 100644 --- a/capsules/object-provider/Cargo.lock +++ b/capsules/object-provider/Cargo.lock @@ -167,9 +167,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arc-swap" @@ -1190,7 +1187,7 @@ dependencies = [ "elastos-protected-content-provider-contracts", "elastos-protected-content-rights", "getrandom 0.2.17", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -1274,17 +1271,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -1589,7 +1575,7 @@ dependencies = [ "elastos-common", "elastos-compute", "libc", - "nix", + "nix 0.29.0", "serde", "serde_json", "thiserror 1.0.69", @@ -1689,7 +1675,7 @@ dependencies = [ "ghash", "hkdf", "hpke", - "nix", + "nix 0.29.0", "rand 0.10.2", "rand 0.9.5", "rand_chacha 0.3.1", @@ -1734,7 +1720,7 @@ dependencies = [ "elastos-protected-content-provider-contracts", "elastos-wallet-contract", "hex", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -3855,6 +3841,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -6053,6 +6051,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -7446,18 +7450,16 @@ dependencies = [ [[package]] name = "zip" -version = "2.4.2" +version = "9.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +checksum = "a954114b47a54df49f83d2722814e86b23d4bb1b54a1b90fb7d92b41619ddc22" dependencies = [ - "arbitrary", "crc32fast", - "crossbeam-utils", - "displaydoc", "flate2", "indexmap", "memchr", - "thiserror 2.0.20", + "nix 0.31.3", + "typed-path", ] [[package]] diff --git a/elastos/Cargo.lock b/elastos/Cargo.lock index ac82b721d..dc06da84e 100644 --- a/elastos/Cargo.lock +++ b/elastos/Cargo.lock @@ -167,9 +167,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arc-swap" @@ -1197,7 +1194,7 @@ dependencies = [ "elastos-protected-content-provider-contracts", "elastos-protected-content-rights", "getrandom 0.2.17", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -1281,17 +1278,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -1598,7 +1584,7 @@ dependencies = [ "elastos-common", "elastos-compute", "libc", - "nix", + "nix 0.29.0", "serde", "serde_json", "tempfile", @@ -1706,7 +1692,7 @@ dependencies = [ "hkdf", "hpke", "k256", - "nix", + "nix 0.29.0", "rand 0.10.2", "rand 0.9.5", "rand_chacha 0.3.1", @@ -1765,7 +1751,7 @@ dependencies = [ "elastos-wallet-contract", "hex", "k256", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -3925,6 +3911,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -6164,6 +6162,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -7703,18 +7707,16 @@ dependencies = [ [[package]] name = "zip" -version = "2.4.2" +version = "9.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +checksum = "a954114b47a54df49f83d2722814e86b23d4bb1b54a1b90fb7d92b41619ddc22" dependencies = [ - "arbitrary", "crc32fast", - "crossbeam-utils", - "displaydoc", "flate2", "indexmap", "memchr", - "thiserror 2.0.20", + "nix 0.31.3", + "typed-path", ] [[package]] diff --git a/elastos/crates/elastos-server/Cargo.toml b/elastos/crates/elastos-server/Cargo.toml index 84788f1d5..e54e95885 100644 --- a/elastos/crates/elastos-server/Cargo.toml +++ b/elastos/crates/elastos-server/Cargo.toml @@ -104,7 +104,7 @@ tempfile = "3.10" # Archive extraction (for capsule artifacts) flate2 = "1.0" tar = "0.4" -zip = { version = "2.4.2", default-features = false, features = ["deflate-flate2", "flate2"] } +zip = { version = "9.0.4", default-features = false, features = ["deflate-flate2"] } libc = "0.2" # Built-in Carrier transport (P2P for updates, gossip messaging, DHT discovery). diff --git a/elastos/crates/elastos-server/src/api/gateway_tests/library.rs b/elastos/crates/elastos-server/src/api/gateway_tests/library.rs index d8477c0db..0253c229f 100644 --- a/elastos/crates/elastos-server/src/api/gateway_tests/library.rs +++ b/elastos/crates/elastos-server/src/api/gateway_tests/library.rs @@ -892,7 +892,7 @@ fn zip_text_files(bytes: &[u8]) -> std::collections::BTreeMap { if entry.is_dir() { continue; } - let name = entry.name().to_string(); + let name = entry.name().unwrap().into_owned(); let mut body = String::new(); entry.read_to_string(&mut body).unwrap(); files.insert(name, body); @@ -3870,6 +3870,203 @@ async fn test_library_provider_rejects_unsafe_zip_entries() { .contains("relative and safe")); } +/// Every path under `root` (files, folders and symlinks, not followed). +fn walk_paths(root: &std::path::Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + stack.push(path.clone()); + } + out.push(path); + } + } + out +} + +#[tokio::test] +async fn test_library_provider_extract_zip_refuses_absolute_dotdot_and_symlink_entries() { + use std::io::Write as _; + + let dir = tempfile::tempdir().unwrap(); + let app = gateway_router(library_test_state(dir.path()).await); + let authority = passkey_authority_with_name(dir.path(), Some("admin")); + let token = app_token_for_authority(dir.path(), LIBRARY_CAPSULE_ID, &authority); + let root = crate::auth::principal_localhost_root(&authority.principal_id); + let documents_uri = format!("{root}/Documents"); + let imports_uri = format!("{documents_uri}/Imports"); + + let (status, _) = post_library( + app.clone(), + &token, + "mkdir", + json!({ "parent_uri": root, "name": "Documents" }), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (status, _) = post_library( + app.clone(), + &token, + "mkdir", + json!({ "parent_uri": documents_uri, "name": "Imports" }), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let options = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + // Each archive holds one hostile entry. The whole-archive extract must + // refuse it and write nothing for it, inside or outside the Library. + // (archive name, entry kind, entry name, symlink target, expected message) + let cases: [(&str, &str, &str, &str, &str); 5] = [ + ( + "Absolute.zip", + "file", + "/absolute-escape.txt", + "", + "relative and safe", + ), + ( + "DotDot.zip", + "file", + "../dotdot-escape.txt", + "", + "relative and safe", + ), + ( + "Inner.zip", + "file", + "Nested/../inner-escape.txt", + "", + "relative and safe", + ), + ( + "Backslash.zip", + "file", + "..\\backslash-escape.txt", + "", + "invalid library object name", + ), + ( + "Symlink.zip", + "symlink", + "symlink-escape.txt", + "/etc/passwd", + "non-file", + ), + ]; + for (archive_name, kind, entry_name, link_target, expected) in cases { + let archive_bytes = { + let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + if kind == "symlink" { + writer + .add_symlink(entry_name, link_target, options) + .unwrap(); + } else { + writer.start_file(entry_name, options).unwrap(); + writer.write_all(b"escape").unwrap(); + } + writer.finish().unwrap().into_inner() + }; + let zip_uri = format!("{documents_uri}/{archive_name}"); + let (write_status, write) = post_library( + app.clone(), + &token, + "write", + json!({ + "uri": zip_uri, + "mime": "application/zip", + "data": base64::engine::general_purpose::STANDARD.encode(archive_bytes), + }), + ) + .await; + assert_eq!(write_status, StatusCode::OK, "{archive_name}"); + assert_eq!(write["status"], "ok", "{archive_name}: {write}"); + + let (extract_status, extracted) = post_library( + app.clone(), + &token, + "extract_archive", + json!({ "uri": zip_uri }), + ) + .await; + assert_eq!(extract_status, StatusCode::OK, "{archive_name}"); + assert_eq!( + extracted["status"], "error", + "{archive_name}: hostile entry {entry_name:?} was not refused: {extracted}" + ); + assert!( + extracted["message"].as_str().unwrap().contains(expected), + "{archive_name}: {extracted}" + ); + + if kind == "symlink" { + // The listing marks it blocked and a selected extract refuses it. + let (list_status, listed) = post_library( + app.clone(), + &token, + "archive_entries", + json!({ "uri": zip_uri }), + ) + .await; + assert_eq!(list_status, StatusCode::OK); + let rows = listed["data"]["entries"].as_array().unwrap(); + assert!( + rows.iter().any(|entry| entry["path"] == entry_name + && entry["kind"] == "blocked" + && entry["safety"]["reason"] + .as_str() + .unwrap() + .contains("non-file")), + "{listed}" + ); + let (select_status, selected) = post_library( + app.clone(), + &token, + "archive_extract_entries", + json!({ + "uri": zip_uri, + "destination_uri": imports_uri, + "entries": [entry_name], + }), + ) + .await; + assert_eq!(select_status, StatusCode::OK); + assert_eq!(selected["status"], "ok", "{selected}"); + assert_eq!( + selected["data"]["receipt"]["progress"]["blocked_entries"], + 1 + ); + assert!(selected["data"]["blocked"][0]["reason"] + .as_str() + .unwrap() + .contains("non-file")); + } + } + + // Nothing from any hostile entry exists anywhere under the data dir, and + // no symlink was created. + for path in walk_paths(dir.path()) { + let name = path.file_name().unwrap().to_string_lossy().to_string(); + assert!( + !name.contains("escape"), + "hostile entry was written: {}", + path.display() + ); + assert!( + !std::fs::symlink_metadata(&path) + .unwrap() + .file_type() + .is_symlink(), + "symlink was created: {}", + path.display() + ); + } +} + #[tokio::test] async fn test_library_gateway_reads_webspace_files_through_runtime_provider() { let dir = tempfile::tempdir().unwrap(); diff --git a/elastos/crates/elastos-server/src/library.rs b/elastos/crates/elastos-server/src/library.rs index 9e537d229..84c791d0a 100644 --- a/elastos/crates/elastos-server/src/library.rs +++ b/elastos/crates/elastos-server/src/library.rs @@ -4239,7 +4239,10 @@ fn preview_zip_archive_entry( let mut file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let raw_name = file.name().to_string(); + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))? + .into_owned(); let Ok(normalized) = normalized_archive_entry_path(Path::new(&raw_name)) else { continue; }; @@ -4457,7 +4460,10 @@ fn list_zip_archive_entries(bytes: Vec) -> anyhow::Result<(Vec, bool) let file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let raw_name = file.name().to_string(); + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))? + .into_owned(); let path = Path::new(&raw_name); let normalized = match normalized_archive_entry_path(path) { Ok(path) => path, @@ -4939,7 +4945,10 @@ fn collect_selected_zip_writes( let mut file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let raw_name = file.name().to_string(); + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))? + .into_owned(); let normalized = match normalized_archive_entry_path(Path::new(&raw_name)) { Ok(path) => path, Err(err) => { @@ -5429,8 +5438,10 @@ fn extract_zip_archive( let mut file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let entry_path = file - .enclosed_name() + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; + let entry_path = zip_entry_enclosed_path(&raw_name) .ok_or_else(|| anyhow!("library archive entry path must be relative and safe"))?; let entry_uri = archive_entry_uri(&destination.uri, &entry_path)?; if entry_uri == destination.uri { @@ -5461,6 +5472,29 @@ fn extract_zip_archive( Ok(()) } +/// The path a whole-archive ZIP extract writes for an entry name, or `None` +/// when the name holds a NUL byte, is absolute (a root or a drive prefix), or +/// climbs above the archive root. `archive_entry_uri` then refuses any `..` +/// that is left. This keeps the rule in the runtime: zip 9 `enclosed_name` +/// strips a leading `/` or drive prefix and folds `..` instead, which would +/// write entries that this extract refuses. +fn zip_entry_enclosed_path(name: &str) -> Option { + if name.contains('\0') { + return None; + } + let path = PathBuf::from(name); + let mut depth = 0usize; + for component in path.components() { + match component { + Component::Prefix(_) | Component::RootDir => return None, + Component::ParentDir => depth = depth.checked_sub(1)?, + Component::Normal(_) => depth += 1, + Component::CurDir => {} + } + } + Some(path) +} + fn archive_entry_uri(destination_uri: &str, path: &Path) -> anyhow::Result { let mut uri = destination_uri.trim_end_matches('/').to_string(); for component in path.components() { From 2e70c052c5699896ba97a4551efd2385c7cb4d8f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 08:24:24 +0000 Subject: [PATCH 2/8] ci: installed journey receipts name the observer failure that stopped the run The finite-memory observer and the model-repair control thread caught every exception and kept only a fixed code, so a failed job said "installed Runtime exited before health" or "finite installed UI journey failed" and nothing else. The uploaded installed-journeys.json now carries the phase (census, stage request, engine stage, deadline), the exception type, a bounded one-line message, and whether the Runtime process had already exited. The same text follows the CI log line. Only harness-written messages go in; the Runtime log stays private. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- scripts/ci-installed-journeys.py | 48 +++++++++++++++++-- scripts/ci-release-policy-test.py | 78 +++++++++++++++++++++++++++++-- 2 files changed, 118 insertions(+), 8 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index 22ac0d41d..705ceedb2 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -57,6 +57,19 @@ def private_json(path, value=None): return json.loads(encoded) +def observer_failure(phase, error, limit=480): + """Public cause of an observer failure. The harness writes these messages; + no observer reads the private Runtime log, so the receipt can carry them.""" + message = " ".join(str(error).split()) + if len(message) > limit: + message = message[:limit] + "..." + return {"phase": phase, "type": type(error).__name__, "message": message} + + +def failure_suffix(failure): + return f" ({failure['phase']}: {failure['type']}: {failure['message']})" if failure else "" + + def linux_process_identity(pid, proc=Path("/proc"), retained=None): """Bind a live process by UID, kernel start tick and executable inode.""" directory = proc / str(pid) @@ -237,6 +250,7 @@ def __init__(self, home, data, address, environment, gib, platform=sys.platform) self.metric_fds = {} self.stop = threading.Event() self.error = None + self.failure = None self.thread = None self.outside = [os.getpid()] self.engine = None @@ -488,24 +502,39 @@ def action(self, request): "engine_pid": self.engine, "engine_start_ticks": self.known[self.engine]["start_ticks"]}) private_json(self.control / "response.json", {"id": self.last_action, "status": "passed", "engine_start_ticks": self.known[self.engine]["start_ticks"], "engine_pid": self.engine}) + def runtime_exited(self): + try: + return self.pid in self.fds and pidfd_exited(self.fds[self.pid]) + except OSError: + return None + def observe(self): + phase = "census" try: while not self.stop.is_set() and time.monotonic() - self.started < 1800: + phase = "census" self.observe_once() path = self.control / "request.json" if path.exists(): + phase = "stage_request" request = private_json(path) if request.get("id") != self.last_action: + phase = "stage_" + str(request.get("action"))[:32] self.action(request) self.stop.wait(.1) if not self.stop.is_set(): + phase = "deadline" raise RuntimeError("finite journey observer deadline") - except BaseException: + except BaseException as error: + # "census" with "Runtime left its unit" and runtime_exited true means + # Runtime itself ended; other census messages name the failed check. + self.failure = {**observer_failure(phase, error), "runtime_exited": self.runtime_exited()} self.error = "finite_resource_observation_failed" private_json(self.control / "response.json", {"id": self.last_action + 1, "status": "failed"}) private_json(self.control / "stop.json", {"stop": True}) def finish(self): + runtime_exited = self.runtime_exited() if self.pid is not None else None self.stop.set() if self.thread: self.thread.join(timeout=3) @@ -513,6 +542,10 @@ def finish(self): "status": "failed", "cgroup_empty": False, "observer_complete": self.thread is None or not self.thread.is_alive(), "actions": self.actions, "outside_runtime_unit": True} + if self.failure: + record["observer_failure"] = self.failure + if runtime_exited: + record["runtime_exited_before_cleanup"] = True phase = "unit_binding" try: if not self.launch_requested: @@ -865,7 +898,7 @@ def free_port(kind=socket.SOCK_STREAM): def wait_healthy(child, address, label, stop=None): deadline = time.monotonic() + 120 while time.monotonic() < deadline and not (stop and stop.is_set()): - assert child.poll() is None, f"installed {label} exited before health" + assert child.poll() is None, f"installed {label} exited before health" + failure_suffix(getattr(child, "failure", None)) try: with urllib.request.urlopen(f"http://{address}/healthz", timeout=1) as response: if response.status == 200: @@ -943,6 +976,7 @@ def __init__(self, home, data, address, environment): self.stop = threading.Event() self.launch_lock = threading.Lock() self.error = None + self.failure = None try: self.launch() self.record = {"runtime_episodes": [self.identity()], "actions": []} @@ -1066,7 +1100,7 @@ def restart_get(self): return {**self.kept, "home_restarted": True} def observe(self): - previous = 0 + previous, phase = 0, "control_request" try: while not self.stop.wait(.05): try: @@ -1080,11 +1114,15 @@ def observe(self): action = request["action"] if action not in ("engine_running", "engine_stopped", "restart_get") or action in self.record["actions"]: raise RuntimeError("repair journey control action differs") + phase = "stage_" + action response = getattr(self, action)() + phase = "control_request" previous = request["id"] self.record["actions"].append(action) private_json(self.control / "response.json", {"id": previous, "status": "passed", **response}) - except BaseException: + except BaseException as error: + self.failure = observer_failure(phase, error) + self.record["control_failure"] = self.failure self.error = "installed_model_repair_control_failed" private_json(self.control / "response.json", {"id": previous + 1, "status": "failed"}) @@ -1258,7 +1296,7 @@ def run(home, data, evidence, model=True, carrier=False, memory_gib=0, repair=Fa while browser_owner.poll() is None and time.monotonic() < deadline and not finite.error: time.sleep(.1) if browser_owner.poll() != 0 or finite.error: - raise RuntimeError("finite installed UI journey failed") + raise RuntimeError("finite installed UI journey failed" + failure_suffix(finite.failure)) finally: if browser_owner.poll() is None: browser_owner.terminate() diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index 4d670a620..71ed8665e 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -15,6 +15,7 @@ import tarfile import tempfile import textwrap +import time import unittest from unittest import mock @@ -2161,14 +2162,15 @@ def consumer_kubo(self, data, holds_root=False): (blocks / key[-3:-1]).mkdir() (blocks / key[-3:-1] / f"{key}.data").write_bytes(b"package root") - def execute(self, home, data, evidence, available=20 * 1024 ** 3, carrier=False, consumer_holds_root=False, memory_gib=0): + def execute(self, home, data, evidence, available=20 * 1024 ** 3, carrier=False, consumer_holds_root=False, memory_gib=0, + finite=None): journey = runpy.run_path(str(WORKFLOW.parents[2] / "scripts/ci-installed-journeys.py")) child = mock.Mock(pid=12345) child.poll.return_value = None response = mock.MagicMock() response.__enter__.return_value.status = 200 self.holder = mock.Mock(return_value=(mock.Mock(pid=23456), {"holder_did": "did:key:holder"})) - self.finite = mock.Mock() + self.finite = finite or mock.Mock() def node_journey(*args, **kwargs): if args[0][1] == "scripts/ci-model-package.mjs": @@ -2701,7 +2703,7 @@ def finite_owner(self, root): owner.engine, owner.engine_members = None, set() owner.stop = mock.Mock() owner.stop.is_set.return_value = False - owner.thread, owner.error, owner.launch_requested = None, None, True + owner.thread, owner.error, owner.failure, owner.launch_requested = None, None, None, True owner.baseline_events = {"oom": 0, "oom_kill": 0} owner.show = mock.Mock(return_value=dict(owner.binding)) owner.command = mock.Mock() @@ -3072,6 +3074,76 @@ def test_finite_engine_stage_refuses_wrong_binding_interruption_and_deadline(sel self.assertEqual(owner.actions, []) self.assertFalse((owner.control / "response.json").exists()) + def test_finite_observer_failure_names_its_phase_type_message_and_runtime_exit(self): + cases = (("census", RuntimeError("finite journey Runtime left its unit"), True), + ("census", RuntimeError("finite journey child escaped its finite unit"), False), + ("stage_engine_running", RuntimeError("finite journey engine stage deadline"), False)) + for phase, error, exited in cases: + with self.subTest(phase=phase, error=str(error)), tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp)) + owner.started, owner.fds = time.monotonic(), {100: 1100} + owner.observe_once = mock.Mock(side_effect=None if phase.startswith("stage") else error) + owner.action = mock.Mock(side_effect=error) + if phase.startswith("stage"): + self.journey["private_json"](owner.control / "request.json", {"id": 1, "action": "engine_running"}) + self.journey["private_json"](owner.control / "runtime-stopped.json", {"reaped": True}) + with mock.patch.dict(self.globals, {"pidfd_exited": lambda fd: exited and fd == 1100}), \ + mock.patch.object(os, "close"): + owner.observe() + expected = {"phase": phase, "type": "RuntimeError", "message": str(error), "runtime_exited": exited} + self.assertEqual(owner.failure, expected) + self.assertEqual(owner.poll(), 1) + # The same cause reaches the CI log line that used to end the job without it. + with self.assertRaisesRegex(AssertionError, re.escape(f"exited before health ({phase}: RuntimeError: {error})")): + self.journey["wait_healthy"](owner, "127.0.0.1:1", "Runtime") + receipt = owner.finish() + self.assertEqual(receipt["status"], "failed") + self.assertEqual(receipt["observer_failure"], expected) + self.assertEqual(receipt.get("runtime_exited_before_cleanup"), True if exited else None) + + def test_observer_failure_text_is_bounded_and_single_line(self): + cause = self.journey["observer_failure"]("census", OSError("first line\nsecond " + "x" * 2000)) + self.assertEqual(cause["type"], "OSError") + self.assertNotIn("\n", cause["message"]) + self.assertTrue(cause["message"].startswith("first line second x")) + self.assertLessEqual(len(cause["message"]), 483) + + def test_finite_observer_failure_reaches_the_uploaded_installed_journey_receipt(self): + with tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp) / "observer") + owner.started, owner.fds, owner.launch_requested = time.monotonic(), {}, False + owner.observe_once = mock.Mock(side_effect=RuntimeError("finite journey Runtime left its unit")) + owner.observe() + owner.start = mock.Mock(return_value=owner) + case = InstalledJourneyTests() + home, data, evidence, _ = case.fixture(Path(temp) / "journey", "linux") + # The unit helper creates the private log before start() returns. + (home / "journey-runtime.private.log").write_text("private Runtime log line\n") + factory = mock.Mock(return_value=owner) + with self.assertRaisesRegex(AssertionError, re.escape( + "installed Runtime exited before health (census: RuntimeError: finite journey Runtime left its unit)")): + case.execute(home, data, evidence, memory_gib=4, finite=factory) + factory.assert_called_once() + record = json.loads((evidence / "installed-journeys.json").read_text()) + self.assertEqual(record["finite_memory"]["observer_failure"]["message"], "finite journey Runtime left its unit") + self.assertEqual(record["finite_memory"]["observer_failure"]["phase"], "census") + self.assertEqual(record["results"]["finite_memory_boundary"], "failed") + self.assertNotIn("private Runtime log line", (evidence / "installed-journeys.json").read_text()) + + def test_repair_control_failure_reaches_its_record(self): + owner = self.journey["RepairRuntime"].__new__(self.journey["RepairRuntime"]) + with tempfile.TemporaryDirectory() as temp: + owner.control, owner.record = Path(temp), {"actions": []} + owner.control.chmod(0o700) + owner.stop = mock.Mock() + owner.stop.wait.return_value = False + self.journey["private_json"](owner.control / "request.json", {"id": 1, "action": "engine_running"}) + owner.engine_running = mock.Mock(side_effect=RuntimeError("repair journey engine deadline")) + owner.observe() + self.assertEqual(owner.error, "installed_model_repair_control_failed") + self.assertEqual(owner.record["control_failure"], {"phase": "stage_engine_running", "type": "RuntimeError", + "message": "repair journey engine deadline"}) + def test_finite_start_requires_actual_runtime_and_every_kernel_limit(self): for fault in (None, "launcher_pid", "wrong_parent", "unit_max", "swap", "tasks", "tighter_parent", "missing_max", "missing_current", "malformed_current"): From 1ba344a60cad0a3d2d3d196ca3561d6a06499796 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 08:29:00 +0000 Subject: [PATCH 3/8] ci: SmolLM2 proof copies the vendored Apache License text instead of downloading it Every installed-journey run downloaded LICENSE-2.0.txt from apache.org, and a 503 there failed ARM CI (run 38899867022). The pinned text now lives at scripts/fixtures/licenses/Apache-2.0.txt with the same size and SHA-256 as the fixture pin. pinned-model-consumer-proof.sh and ci-model-package.mjs produce check the vendored bytes against the pin, copy them, and check the copy. A changed vendored file stops the run before the model download, and there is no network fallback. The Qwen fixture keeps its pinned Hugging Face revision. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- scripts/ci-model-package.mjs | 20 ++- scripts/ci-model-package.test.mjs | 26 ++- scripts/ci-release-policy-test.py | 44 +++++ scripts/fixtures/licenses/Apache-2.0.txt | 202 +++++++++++++++++++++++ scripts/pinned-model-consumer-proof.sh | 28 +++- scripts/pinned-smollm2-fixture.json | 2 +- 6 files changed, 308 insertions(+), 14 deletions(-) create mode 100644 scripts/fixtures/licenses/Apache-2.0.txt diff --git a/scripts/ci-model-package.mjs b/scripts/ci-model-package.mjs index 2d62dc955..6b977717d 100644 --- a/scripts/ci-model-package.mjs +++ b/scripts/ci-model-package.mjs @@ -41,7 +41,8 @@ export const rawCid = bytes => "b" + encode(Buffer.concat([Buffer.from([1, 0x55, // PROVENANCE.md is checked against a fixed charset before any download. const FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; const REPOSITORY = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/; -const APACHE_LICENSE_URL = "https://www.apache.org/licenses/LICENSE-2.0.txt"; +// The Apache License 2.0 text is vendored beside the fixtures; it has no network source. +const APACHE_LICENSE_PATH = "fixtures/licenses/Apache-2.0.txt"; export function loadFixture(path) { const fixture = JSON.parse(readFileSync(path, "utf8")); const manifest = fixture.capsule_manifest, model = manifest?.model_content, provenance = model?.provenance; @@ -57,7 +58,9 @@ export function loadFixture(path) { } const pinned = `https://huggingface.co/${provenance.quantized_repository}/resolve/${provenance.quantized_revision}/`; assert.equal(fixture.model.url, pinned + fixture.model.name, "fixture weights must come from the pinned quantized repository revision"); - assert([pinned + "LICENSE", APACHE_LICENSE_URL].includes(fixture.license.url), "fixture license must come from the pinned revision or apache.org"); + assert(fixture.license.url === undefined ? fixture.license.path === APACHE_LICENSE_PATH + : fixture.license.path === undefined && fixture.license.url === pinned + "LICENSE", + "fixture license must come from the pinned revision or the vendored Apache License text"); assert(["Q1_0", "Q4_K_M", "Q8_0"].includes(model.quantization), "fixture quantization is unsupported"); assert(model.license.spdx_id === "Apache-2.0" && provenance.base_license.spdx_id === "Apache-2.0", "fixture licenses must be Apache-2.0"); assert(/^[a-z0-9][a-z0-9._-]{0,127}$/.test(manifest.name), "fixture capsule name must be a plain identifier"); @@ -291,6 +294,15 @@ export async function download(source, path, get = fetch) { } } +// A vendored pin is checked in the repository before it is copied, and the +// copy is checked again. A changed vendored file stops the run. +export function copyVendored(source, path, base = join(root, "scripts")) { + if (existsSync(path)) return fileRecord(path, source.name, source); + fileRecord(join(base, source.path), source.name, source); + copyFileSync(join(base, source.path), path, constants.COPYFILE_EXCL); + return fileRecord(path, source.name, source); +} + // usage: produce // is a data dir prepared by scripts/seed-kubo-cache.sh (pinned recipe Kubo). async function produce(args) { @@ -302,8 +314,10 @@ async function produce(args) { assert.deepEqual(fileRecord(kubo, entrypoint.path), entrypoint, "Kubo binary matches its pinned recipe build"); const inputs = ownedDirectory(join(ownedDirectory(output), "inputs")), weights = join(inputs, fixture.model.name); assert.equal(dirname(weights), inputs, "fixture weights stay inside OUT/inputs"); + // The local license check runs first, so a changed copy stops before the model download. + if (fixture.license.path) copyVendored(fixture.license, join(inputs, "LICENSE")); + else await download(fixture.license, join(inputs, "LICENSE")); await download(fixture.model, weights); - await download(fixture.license, join(inputs, "LICENSE")); const repo = mkdtempSync(join(output, ".ipfs-repo-")); try { const { cid, bytes, packageDir } = producePackage({ inputs, output, publisher, fixture, add: dir => kuboAdd(kubo, repo, dir) }); diff --git a/scripts/ci-model-package.test.mjs b/scripts/ci-model-package.test.mjs index 231257669..50e31f8de 100644 --- a/scripts/ci-model-package.test.mjs +++ b/scripts/ci-model-package.test.mjs @@ -3,8 +3,8 @@ import test from "node:test"; import { createHash, createPublicKey, verify } from "node:crypto"; import { chmodSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { buildEntry, canonical, contentManifest, download, fileRecord, loadFixture, ownedDirectory, producePackage, rawCid, signedCatalog, SMOL_FIXTURE, verifyInstalledKubo, writeCatalogPayload } from "./ci-model-package.mjs"; +import { dirname, join } from "node:path"; +import { buildEntry, canonical, contentManifest, copyVendored, download, fileRecord, loadFixture, ownedDirectory, producePackage, rawCid, signedCatalog, SMOL_FIXTURE, verifyInstalledKubo, writeCatalogPayload } from "./ci-model-package.mjs"; const QWEN_FIXTURE = loadFixture(new URL("./pinned-qwen2.5-1.5b-fixture.json", import.meta.url)); @@ -64,7 +64,10 @@ test("fixture loading refuses escaping names, injected text, unpinned weights an refused(f => { provenance(f).quantized_repository = "Qwen/Qwen2.5\nSigned by someone else."; }, /owner\/name identifiers/); refused(f => { provenance(f).base_repository = "Qwen/../../etc"; }, /owner\/name identifiers/); refused(f => { f.model.url = f.model.url.replace(/resolve\/[0-9a-f]+\//, "resolve/main/"); }, /pinned quantized repository revision/); - refused(f => { f.license.url = "https://example.com/LICENSE"; }, /pinned revision or apache\.org/); + refused(f => { f.license.url = "https://example.com/LICENSE"; }, /pinned revision or the vendored Apache License text/); + refused(f => { f.license.url = "https://www.apache.org/licenses/LICENSE-2.0.txt"; }, /pinned revision or the vendored Apache License text/); + refused(f => { delete f.license.url; f.license.path = "../../outside/LICENSE"; }, /pinned revision or the vendored Apache License text/); + refused(f => { f.license.path = "fixtures/licenses/Apache-2.0.txt"; }, /pinned revision or the vendored Apache License text/); refused(f => { provenance(f).quantized_revision = "91cad511"; }, /full commit hashes/); refused(f => { f.capsule_manifest.description = "one\nline"; }, /one bounded line/); refused(f => { f.display_name = "Qwen 1.5B"; }, /display_name/); @@ -227,11 +230,26 @@ test("download refuses a pre-placed symlink partial and leaves its target untouc symlinkSync(victim, join(inputs, "LICENSE.partial")); let fetched = false; const get = async () => { fetched = true; return new Response("replaced"); }; - await assert.rejects(download(SMOL_FIXTURE.license, join(inputs, "LICENSE"), get), /EEXIST/); + await assert.rejects(download(QWEN_FIXTURE.license, join(inputs, "LICENSE"), get), /EEXIST/); assert.equal(readFileSync(victim, "utf8"), "keep"); assert.equal(fetched, false); }); +test("the SmolLM2 license is the vendored Apache text, copied only after its pin matches", t => { + assert.equal(SMOL_FIXTURE.license.url, undefined, "the SmolLM2 license has no network source"); + const root = temporary(t), base = join(root, "scripts"); + mkdirSync(join(root, "good")); mkdirSync(join(root, "tampered")); + const copied = copyVendored(SMOL_FIXTURE.license, join(root, "good/LICENSE")); + assert.deepEqual(copied, { path: "LICENSE", size: SMOL_FIXTURE.license.size, sha256: SMOL_FIXTURE.license.sha256 }); + mkdirSync(dirname(join(base, SMOL_FIXTURE.license.path)), { recursive: true }); + const text = readFileSync(join(root, "good/LICENSE"), "utf8"); + for (const [name, changed] of [["same size", text.replace("Apache License", "Apache Licanse")], ["appended", text + "extra grant\n"]]) { + writeFileSync(join(base, SMOL_FIXTURE.license.path), changed); + assert.throws(() => copyVendored(SMOL_FIXTURE.license, join(root, "tampered/LICENSE"), base), /pinned (checksum|size)/, name); + assert.throws(() => lstatSync(join(root, "tampered/LICENSE")), /ENOENT/, name); + } +}); + test("output directory refuses a symlinked ancestor", t => { const root = temporary(t), real = join(root, "real"); mkdirSync(real); diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index 71ed8665e..fe4d221b9 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -3754,5 +3754,49 @@ def test_observer_refuses_unrelated_processes_and_foreign_engine_paths(self): self.assertEqual(list(observe({**rows, 3: (2, guard)}, 1, data)), []) +class PinnedModelProofLicenseTests(unittest.TestCase): + """The proof copies the vendored license text only after its pin matches; it never downloads it.""" + def proof(self, root, change=None): + tree = root / "tree" + (tree / "scripts/fixtures/licenses").mkdir(parents=True) + for name in ("pinned-model-consumer-proof.sh", "pinned-smollm2-fixture.json", "fixtures/licenses/Apache-2.0.txt"): + shutil.copy2(REPO / "scripts" / name, tree / "scripts" / name) + vendored = tree / "scripts/fixtures/licenses/Apache-2.0.txt" + if change: + vendored.write_bytes(change(vendored.read_bytes())) + tools = root / "tools" + tools.mkdir() + (tools / "curl").write_text("#!/bin/sh\ntouch \"$0.called\"\nexit 22\n") + (tools / "curl").chmod(0o700) + (root / "model.gguf").write_bytes(b"not the pinned model") + environment = {"PATH": f"{tools}:/usr/bin:/bin", "HOME": str(root), + "ELASTOS_PROOF_MODEL_PATH": str(root / "model.gguf")} + result = subprocess.run(["bash", str(tree / "scripts/pinned-model-consumer-proof.sh"), str(root / "proof"), + "--fetch", "--check-inputs"], env=environment, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60) + self.assertFalse((tools / "curl.called").exists(), "the license has no network source") + return result, root / "proof/inputs/LICENSE" + + def test_vendored_license_matches_its_pin_and_is_copied_before_model_checks(self): + with tempfile.TemporaryDirectory() as temp: + result, copied = self.proof(Path(temp)) + fixture = json.loads((REPO / "scripts/pinned-smollm2-fixture.json").read_text())["license"] + self.assertNotIn("url", fixture) + self.assertIn(f"vendored license verified: {fixture['size']} bytes sha256 {fixture['sha256']}", result.stdout) + self.assertEqual(hashlib.sha256(copied.read_bytes()).hexdigest(), fixture["sha256"]) + # The dummy model is refused after the license step; nothing else runs. + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("model size", result.stdout) + + def test_tampered_vendored_license_is_refused_before_copy_or_download(self): + for name, change in (("same size", lambda text: text.replace(b"Apache License", b"Apache Licanse", 1)), + ("appended", lambda text: text + b"extra grant\n"), ("empty", lambda text: b"")): + with self.subTest(change=name), tempfile.TemporaryDirectory() as temp: + result, copied = self.proof(Path(temp), change) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertRegex(result.stdout, r"vendored license (sha256|size) .* differs from pinned") + self.assertFalse(copied.exists() or copied.is_symlink()) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/fixtures/licenses/Apache-2.0.txt b/scripts/fixtures/licenses/Apache-2.0.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/scripts/fixtures/licenses/Apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/scripts/pinned-model-consumer-proof.sh b/scripts/pinned-model-consumer-proof.sh index 32d414e55..12ffd742a 100755 --- a/scripts/pinned-model-consumer-proof.sh +++ b/scripts/pinned-model-consumer-proof.sh @@ -30,7 +30,8 @@ Arguments: New or existing directory owned by this proof. It receives home/ (isolated HOME with the installed engine), kubo/, inputs/ (with --fetch) and proof/ (logs and receipt.json). - --fetch Download the pinned model and license into /inputs + --fetch Download the pinned model into /inputs and copy + the vendored license text there after checking its pin, when ELASTOS_PROOF_MODEL_PATH / _LICENSE_PATH are unset. --native-only Skip the cold Content proof (no Kubo or ipfs-provider). --check-inputs Verify pinned inputs and print the plan; run nothing else. @@ -82,7 +83,7 @@ fields = { "url": fixture["model"]["url"], "bytes": fixture["model"]["size"], "sha256": fixture["model"]["sha256"], - "license_url": fixture["license"]["url"], + "license_path": fixture["license"]["path"], "license_bytes": fixture["license"]["size"], "license_sha256": fixture["license"]["sha256"], "env_prefix": fixture["proof"]["env_prefix"], @@ -127,6 +128,18 @@ fetch_pinned() { chmod 0400 "$dest" } +# The license text lives in the repository beside its fixture pin and has no +# network source. Its bytes are checked before they are copied. +copy_vendored() { + local label="$1" source="$2" dest="$3" bytes="$4" sha="$5" + verify_pinned_file "vendored $label" "$source" "$bytes" "$sha" + if [[ ! -f "$dest" ]]; then + cp -- "$source" "${dest}.partial" + mv -f -- "${dest}.partial" "$dest" + fi + chmod 0400 "$dest" +} + PROOF_MODULE="api::capsule_inventory::preparation::tests::process_proof" # A cargo exit status of zero does not prove that the intended test ran: a @@ -269,14 +282,17 @@ mkdir -p "${PROOF_ROOT}/proof" "${PROOF_ROOT}/inputs" MODEL_PATH="${ELASTOS_PROOF_MODEL_PATH:-}" LICENSE_PATH="${ELASTOS_PROOF_LICENSE_PATH:-}" if [[ "$FETCH" == "1" ]]; then + # The local license check runs first, so a changed copy stops the run + # before the model download starts. + [[ -n "$LICENSE_PATH" ]] || { + LICENSE_PATH="${PROOF_ROOT}/inputs/LICENSE" + copy_vendored "license" "${ROOT}/scripts/$(pinned_field "$MODEL" license_path)" "$LICENSE_PATH" \ + "$(pinned_field "$MODEL" license_bytes)" "$(pinned_field "$MODEL" license_sha256)" + } [[ -n "$MODEL_PATH" ]] || { MODEL_PATH="${PROOF_ROOT}/inputs/$(pinned_field "$MODEL" file)" fetch_pinned "$(pinned_field "$MODEL" url)" "$MODEL_PATH" } - [[ -n "$LICENSE_PATH" ]] || { - LICENSE_PATH="${PROOF_ROOT}/inputs/LICENSE" - fetch_pinned "$(pinned_field "$MODEL" license_url)" "$LICENSE_PATH" - } fi [[ -n "$MODEL_PATH" && -n "$LICENSE_PATH" ]] \ || die "set ELASTOS_PROOF_MODEL_PATH and ELASTOS_PROOF_LICENSE_PATH, or pass --fetch" diff --git a/scripts/pinned-smollm2-fixture.json b/scripts/pinned-smollm2-fixture.json index cd16ff211..18d615574 100644 --- a/scripts/pinned-smollm2-fixture.json +++ b/scripts/pinned-smollm2-fixture.json @@ -9,7 +9,7 @@ }, "license": { "name": "LICENSE", - "url": "https://www.apache.org/licenses/LICENSE-2.0.txt", + "path": "fixtures/licenses/Apache-2.0.txt", "size": 11358, "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" }, From 493f40edb30fe8e339a0a6a8120922c74195fc9d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 12:06:03 +0000 Subject: [PATCH 4/8] ci: observer failure causes drop local paths and token-like values before the receipt is saved observer_failure() copied str(error) into installed-journeys.json and the CI log line. An OSError names the file it failed on, so the runner home, the journey Home and its data folder could reach an uploaded artifact. One redact() now runs on the message before it is bounded. It replaces the journey Home, its data folder (which can hold a space on macOS) and $HOME first, then file: URLs, ~ paths, drive paths and any other absolute path, with . Only a last part that looks like a harness file (request.json, memory.events) stays. It also replaces the value after Bearer/Basic/Token and after secret-named keys (token=, password:), the key part of a DID, and long hex/base64/base58 runs. Phase, type and runtime_exited are unchanged. A new policy test puts a home path, a data path with a space, other absolute paths and tokens in both the finite observer and the repair control failure, and checks the saved receipt, the repair record and the CI log line hold none of them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- scripts/ci-installed-journeys.py | 59 +++++++++++++++++++++++++++---- scripts/ci-release-policy-test.py | 56 +++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 6 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index 705ceedb2..63c3e6b3e 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -57,10 +57,57 @@ def private_json(path, value=None): return json.loads(encoded) -def observer_failure(phase, error, limit=480): - """Public cause of an observer failure. The harness writes these messages; - no observer reads the private Runtime log, so the receipt can carry them.""" - message = " ".join(str(error).split()) +# A kept last path part must look like a harness file (request.json, +# memory.events). Any other last part can be a user, device or key name. +KEPT_PATH_SUFFIXES = {"json", "log", "txt", "events", "max", "current", "peak", "stat", "status", + "service", "sock", "lock", "pid", "bin", "data", "gguf", "car", "toml", "mjs", "py", "sh"} +SECRET_NAME = r"[A-Za-z0-9_.-]*(?:token|secret|passw(?:or)?d|passphrase|api[_-]?key|private[_-]?key|credential|cookie|session|signature|auth)[A-Za-z0-9_.-]*" +PATH_END = "\\s'\"`,;()\\[\\]{}<>" +REDACTIONS = ( + # Credentials after an auth scheme or a secret-named key (token=..., password: ...). + (re.compile(r"\b(Bearer|Basic|Token)\s+[^\s'\",;]+", re.I), r"\1 "), + (re.compile(rf"\b({SECRET_NAME})(\s*[=:]\s*)(?!(?:Bearer|Basic|Token)\s)(['\"]?)[^\s'\",;]+\3", re.I), r"\1\2"), + # A DID names a key, so a person or a device: keep only its method. + (re.compile(r"\bdid:([a-z0-9]+):[^\s'\",;]+"), r"did:\1:"), + # Long hex, base64 or base58 runs: digests, keys, tokens, CIDs, unit and session ids. + (re.compile(r"(?"), +) + + +def redacted_path(path): + name = re.split(r"[/\\]", path.rstrip("/\\"))[-1] + found = re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\.([A-Za-z0-9]{1,8})", name) + return "/" + name if found and found[1].lower() in KEPT_PATH_SUFFIXES else "" + + +def redact(text, roots=()): + """Remove local paths, home folders and token-like values from text that + leaves the runner (receipts and CI log lines).""" + known = set() + for root in (*roots, os.environ.get("HOME")): + if root: + known.update({str(root), os.path.realpath(root)}) + try: + known.add(str(Path.home())) + except (KeyError, RuntimeError): + pass + # Known roots first and longest first: a home or data folder may hold a space. + for root in sorted((root.rstrip("/") for root in known if len(root.rstrip("/")) > 1), key=len, reverse=True): + text = re.sub(re.escape(root) + rf"(?![^/{PATH_END}])[^{PATH_END}]*", lambda found: redacted_path(found[0]), text) + for pattern in (rf"\bfile:/+[^{PATH_END}]*", rf"(?/-])/(?![/\s])[^{PATH_END}]*"): + text = re.sub(pattern, lambda found: redacted_path(found[0]), text) + for pattern, replacement in REDACTIONS: + text = pattern.sub(replacement, text) + return text + + +def observer_failure(phase, error, roots=(), limit=480): + """Public cause of an observer failure. The harness writes these messages + and no observer reads the private Runtime log; the text is still redacted, + because an OSError names local paths.""" + message = redact(" ".join(str(error).split()), roots) if len(message) > limit: message = message[:limit] + "..." return {"phase": phase, "type": type(error).__name__, "message": message} @@ -528,7 +575,7 @@ def observe(self): except BaseException as error: # "census" with "Runtime left its unit" and runtime_exited true means # Runtime itself ended; other census messages name the failed check. - self.failure = {**observer_failure(phase, error), "runtime_exited": self.runtime_exited()} + self.failure = {**observer_failure(phase, error, (self.home, self.data)), "runtime_exited": self.runtime_exited()} self.error = "finite_resource_observation_failed" private_json(self.control / "response.json", {"id": self.last_action + 1, "status": "failed"}) private_json(self.control / "stop.json", {"stop": True}) @@ -1121,7 +1168,7 @@ def observe(self): self.record["actions"].append(action) private_json(self.control / "response.json", {"id": previous, "status": "passed", **response}) except BaseException as error: - self.failure = observer_failure(phase, error) + self.failure = observer_failure(phase, error, (self.home, self.data)) self.record["control_failure"] = self.failure self.error = "installed_model_repair_control_failed" private_json(self.control / "response.json", {"id": previous + 1, "status": "failed"}) diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index fe4d221b9..55803bf69 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -3130,10 +3130,66 @@ def test_finite_observer_failure_reaches_the_uploaded_installed_journey_receipt( self.assertEqual(record["results"]["finite_memory_boundary"], "failed") self.assertNotIn("private Runtime log line", (evidence / "installed-journeys.json").read_text()) + def test_observer_failure_redacts_home_paths_and_tokens_before_the_record_is_saved(self): + hexdigest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + did = "z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK" + private = ["/home/ci-person", "ci-person", "observer root", "Application Support", "/opt/build-cache", + "eyJhbGciOiJFZERTQSJ9.payload-77", "s3cr3t-value", "hunter2pass", hexdigest, did, + "QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz"] + + def message(home, data): + return (f"[Errno 2] No such file or directory: '{data / 'finite-control/request.json'}' " + f"after {home}/model-inputs and /home/ci-person/.config/elastos/keys and " + f"/opt/build-cache/run-41/llama.bin and file:///home/ci-person/x; " + "Authorization: Bearer eyJhbGciOiJFZERTQSJ9.payload-77 token=s3cr3t-value " + f"password: hunter2pass digest {hexdigest} holder did:key:{did} " + "blob QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz") + + def assert_redacted(text, failure): + for value in private: + self.assertNotIn(value, text) + self.assertIn("No such file or directory: '/request.json'", failure["message"]) + self.assertIn("did:key:", failure["message"]) + self.assertIn("Bearer ", failure["message"]) + + with tempfile.TemporaryDirectory() as temp, mock.patch.dict(os.environ, {"HOME": "/home/ci-person"}): + owner, _ = self.finite_owner(Path(temp) / "observer root") + owner.data = owner.home / "Library/Application Support/elastos" + owner.started, owner.fds, owner.launch_requested = time.monotonic(), {}, False + owner.observe_once = mock.Mock(side_effect=FileNotFoundError(message(owner.home, owner.data))) + owner.observe() + owner.start = mock.Mock(return_value=owner) + case = InstalledJourneyTests() + home, data, evidence, _ = case.fixture(Path(temp) / "journey", "linux") + (home / "journey-runtime.private.log").write_text("") + with self.assertRaisesRegex(AssertionError, "installed Runtime exited before health") as raised: + case.execute(home, data, evidence, memory_gib=4, finite=mock.Mock(return_value=owner)) + saved = (evidence / "installed-journeys.json").read_text() + failure = json.loads(saved)["finite_memory"]["observer_failure"] + self.assertEqual((failure["phase"], failure["type"], failure["runtime_exited"]), + ("census", "FileNotFoundError", False)) + assert_redacted(saved, failure) + # The CI log line carries the same redacted cause. + assert_redacted(str(raised.exception), failure) + + repair = self.journey["RepairRuntime"].__new__(self.journey["RepairRuntime"]) + repair.home = owner.home + repair.data = owner.data + repair.control, repair.record = Path(temp) / "repair-control", {"actions": []} + repair.control.mkdir(mode=0o700) + repair.stop = mock.Mock() + repair.stop.wait.return_value = False + self.journey["private_json"](repair.control / "request.json", {"id": 1, "action": "engine_running"}) + repair.engine_running = mock.Mock(side_effect=OSError(message(repair.home, repair.data))) + repair.observe() + self.assertEqual(repair.record["control_failure"]["phase"], "stage_engine_running") + assert_redacted(json.dumps(repair.record), repair.record["control_failure"]) + def test_repair_control_failure_reaches_its_record(self): owner = self.journey["RepairRuntime"].__new__(self.journey["RepairRuntime"]) with tempfile.TemporaryDirectory() as temp: owner.control, owner.record = Path(temp), {"actions": []} + owner.home, owner.data = Path(temp), Path(temp) / "data" owner.control.chmod(0o700) owner.stop = mock.Mock() owner.stop.wait.return_value = False From 269bbe192f7cd71593755ffd721fccb483046dd3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 15:09:45 +0000 Subject: [PATCH 5/8] ci: observer failure causes also drop URL user info, Windows network paths and paths with spaces redact() kept the user and password in scheme://user:pass@host, did not know \\server\share, //server/share, \\?\C:\ or \\.\pipe\ paths, and cut an unknown path at its first space, so "/Volumes/Backup Disk/ci-person" left "Disk/ci-person" in the receipt. The user part of any URL is now replaced (http://@127.0.0.1:8090/). Network, device and drive paths (either slash) start a path. A path now runs until a mark that ends a path in an error message: a quote, a bracket, a comma, a semicolon, a pipe, a line end, or a colon followed by a space. Spaces inside stay in the path. A path with no such mark after it takes the rest of the text: losing words is better than leaking a folder name. A new policy test checks 13 such causes (all 13 fail on the old rules) and that a plain URL and a cause with no path are kept. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- scripts/ci-installed-journeys.py | 29 +++++++++++++++++---- scripts/ci-release-policy-test.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index 63c3e6b3e..26970cc62 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -62,7 +62,27 @@ def private_json(path, value=None): KEPT_PATH_SUFFIXES = {"json", "log", "txt", "events", "max", "current", "peak", "stat", "status", "service", "sock", "lock", "pid", "bin", "data", "gguf", "car", "toml", "mjs", "py", "sh"} SECRET_NAME = r"[A-Za-z0-9_.-]*(?:token|secret|passw(?:or)?d|passphrase|api[_-]?key|private[_-]?key|credential|cookie|session|signature|auth)[A-Za-z0-9_.-]*" -PATH_END = "\\s'\"`,;()\\[\\]{}<>" +# A path runs until a mark that ends a path in an error message: a quote, +# a bracket, a comma, a semicolon, a pipe, a line end, or a colon followed by +# a space or the end ("open /a b/c: denied"). Spaces inside the path stay in +# it, so a folder name with a space is redacted whole ("/Volumes/Backup Disk"). +# A path that is not quoted or followed by such a mark takes the rest of the +# text with it: losing words is better than leaking a folder name. +PATH_STOP = "\\r\\n'\"`,;()\\[\\]{}<>|" +PATH_TAIL = rf"(?:[^{PATH_STOP}\s:]|:(?!\s|$)|[ \t]+(?=[^{PATH_STOP}\s:]|:(?!\s|$)))*" +PATH_START = "|".join(( + r"\bfile:/+", + r"(?/\\-])/(?![/\s])", +)) +# The user part of a URL (scheme://user:password@host) is a credential. +URL_USERINFO = re.compile(r"\b([A-Za-z][A-Za-z0-9+.-]*://)[^\s/?#'\"`<>]*@") REDACTIONS = ( # Credentials after an auth scheme or a secret-named key (token=..., password: ...). (re.compile(r"\b(Bearer|Basic|Token)\s+[^\s'\",;]+", re.I), r"\1 "), @@ -92,12 +112,11 @@ def redact(text, roots=()): known.add(str(Path.home())) except (KeyError, RuntimeError): pass + text = URL_USERINFO.sub(r"\1@", text) # Known roots first and longest first: a home or data folder may hold a space. for root in sorted((root.rstrip("/") for root in known if len(root.rstrip("/")) > 1), key=len, reverse=True): - text = re.sub(re.escape(root) + rf"(?![^/{PATH_END}])[^{PATH_END}]*", lambda found: redacted_path(found[0]), text) - for pattern in (rf"\bfile:/+[^{PATH_END}]*", rf"(?/-])/(?![/\s])[^{PATH_END}]*"): - text = re.sub(pattern, lambda found: redacted_path(found[0]), text) + text = re.sub(re.escape(root) + rf"(?![^/\\{PATH_STOP}\s:]){PATH_TAIL}", lambda found: redacted_path(found[0]), text) + text = re.sub(rf"(?:{PATH_START}){PATH_TAIL}", lambda found: redacted_path(found[0]), text) for pattern, replacement in REDACTIONS: text = pattern.sub(replacement, text) return text diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index 55803bf69..dca12ae16 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -3185,6 +3185,48 @@ def assert_redacted(text, failure): self.assertEqual(repair.record["control_failure"]["phase"], "stage_engine_running") assert_redacted(json.dumps(repair.record), repair.record["control_failure"]) + def test_observer_failure_redacts_url_user_info_windows_network_paths_and_paths_with_spaces(self): + # Each case: the error text, the values that must not survive, and the + # cause that is saved. No case is under a known root, so only the + # general rules can catch it. + cases = ( + ("GET http://ci-user:hunter9@127.0.0.1:8090/api failed", ("ci-user", "hunter9"), + "GET http://@127.0.0.1:8090/api failed"), + ("proxy https://alice@proxy.corp.example:3128 refused", ("alice",), + "proxy https://@proxy.corp.example:3128 refused"), + ("fetch ftp://bob:pw-77@files.example/x (timeout)", ("bob", "pw-77"), + "fetch ftp://@files.example/x (timeout)"), + ("via socks5h://carol:p@ss@10.0.0.1:1080/", ("carol", "p@ss", "ss@"), + "via socks5h://@10.0.0.1:1080/"), + (r"open \\fileserver\share$\ci person\keys.json: denied", ("fileserver", "share$", "person"), + "open /keys.json: denied"), + ("open //fileserver/share/ci person/keys failed", ("fileserver", "share", "person"), "open "), + (r"\\?\C:\Users\ci person\AppData\x.json missing", ("Users", "person", "AppData"), ""), + (r"pipe \\.\pipe\elastos-ci-person busy", ("ci-person",), "pipe "), + (r"C:\Program Files\ElastOS Person\bin\elastos.exe: not found", ("Program", "Person"), + ": not found"), + ("C:/Users/ci person/AppData/Local/elastos/data failed", ("Users", "person"), ""), + ("cannot read /srv/ci person/Shared Keys/vault: Permission denied", ("srv", "person", "Shared", "Keys"), + "cannot read : Permission denied"), + ("No such file or directory: '/var/my folder/secret stuff/key.pem'", ("folder", "secret", "stuff"), + "No such file or directory: ''"), + ("error at /Volumes/Backup Disk/ci-person (exit 2)", ("Volumes", "Backup", "Disk", "ci-person"), + "error at (exit 2)"), + ) + with mock.patch.dict(os.environ, {"HOME": "/home/runner"}): + for text, private, saved in cases: + with self.subTest(text=text): + failure = self.journey["observer_failure"]("census", OSError(text)) + for value in private: + self.assertNotIn(value, failure["message"]) + self.assertEqual(failure["message"], saved) + # The rules keep what carries no person: a plain URL, a quoted + # command's options and a cause with no path. + for text in ("GET http://127.0.0.1:8090/api/health returned 503", "Expecting value: line 1 column 1 (char 0)", + "3/4 samples and/or a // b"): + with self.subTest(text=text): + self.assertEqual(self.journey["observer_failure"]("census", OSError(text))["message"], text) + def test_repair_control_failure_reaches_its_record(self): owner = self.journey["RepairRuntime"].__new__(self.journey["RepairRuntime"]) with tempfile.TemporaryDirectory() as temp: From d241d0f6e8513be03c3b801dfc54a2f3fd29a4b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 00:43:33 +0000 Subject: [PATCH 6/8] ci: observer failure causes drop secret values under quoted and JSON-escaped keys redact() kept the value in {"token":"x"}, {'password': 'p'} and \"token\":\"x\", because the secret-key rule only matched a bare key. One rule (SECRET_VALUE) now matches a bare, quoted or JSON-escaped secret-named key before = or :, and redacts a quoted value up to its own closing quote (or the line end when there is none). The auth-scheme rule no longer takes "=" as the value in "token = x", which left x. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- scripts/ci-installed-journeys.py | 13 +++++++++++-- scripts/ci-release-policy-test.py | 32 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index d7bfd831d..85d05804a 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -95,10 +95,19 @@ def private_json(path, value=None): )) # The user part of a URL (scheme://user:password@host) is a credential. URL_USERINFO = re.compile(r"\b([A-Za-z][A-Za-z0-9+.-]*://)[^\s/?#'\"`<>]*@") +# The value of a secret-named key. The key may be bare, quoted or +# JSON-escaped (token=x, "token": "x", 'password': 'p', \"token\":\"x\"). +# A quoted value ends at the same quote with no backslash before it, so an +# escaped quote inside it stays redacted; with no closing quote the value +# takes the rest of the line. A bare value ends at a space, a quote, a comma, +# a semicolon or a closing bracket. +SECRET_VALUE = re.compile( + rf"""\b({SECRET_NAME})(\\*["']?\s*[=:]\s*)""" + r"""(?:(\\*["'])(?:.*?(?"), - (re.compile(rf"\b({SECRET_NAME})(\s*[=:]\s*)(?!(?:Bearer|Basic|Token)\s)(['\"]?)[^\s'\",;]+\3", re.I), r"\1\2"), + (re.compile(r"\b(Bearer|Basic|Token)\s+(?![=:])[^\s'\",;]+", re.I), r"\1 "), + (SECRET_VALUE, lambda found: found[1] + found[2] + ((found[3] or "") + "" + (found[3] or ""))), # A DID names a key, so a person or a device: keep only its method. (re.compile(r"\bdid:([a-z0-9]+):[^\s'\",;]+"), r"did:\1:"), # Long hex, base64 or base58 runs: digests, keys, tokens, CIDs, unit and session ids. diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index 9a546047f..b3c547c96 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -3331,7 +3331,7 @@ def test_observer_failure_redacts_home_paths_and_tokens_before_the_record_is_sav did = "z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK" private = ["/home/ci-person", "ci-person", "observer root", "Application Support", "/opt/build-cache", "eyJhbGciOiJFZERTQSJ9.payload-77", "s3cr3t-value", "hunter2pass", hexdigest, did, - "QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz"] + "QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz", "q-secret-9"] def message(home, data): return (f"[Errno 2] No such file or directory: '{data / 'finite-control/request.json'}' " @@ -3339,7 +3339,7 @@ def message(home, data): f"/opt/build-cache/run-41/llama.bin and file:///home/ci-person/x; " "Authorization: Bearer eyJhbGciOiJFZERTQSJ9.payload-77 token=s3cr3t-value " f"password: hunter2pass digest {hexdigest} holder did:key:{did} " - "blob QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz") + "blob QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz body {\"token\": \"q-secret-9\"}") def assert_redacted(text, failure, finite=False): for value in private: @@ -3427,6 +3427,34 @@ def test_observer_failure_redacts_url_user_info_windows_network_paths_and_paths_ with self.subTest(text=text): self.assertEqual(self.journey["observer_failure"]("census", OSError(text))["message"], text) + def test_observer_failure_redacts_secret_values_under_quoted_and_escaped_keys(self): + # Each case: the error text, the secret that must not survive, and the + # saved cause. The key and the names around it stay readable. + cases = ( + # The reviewer's case: one letter, so only the exact saved text proves it. + ('{"token":"x"}', None, '{"token":""}'), + ('{"api_key": "abc9"}', "abc9", '{"api_key": ""}'), + ("{'password': 'p4ss'}", "p4ss", "{'password': ''}"), + ('cause "secret" : "s-9 9" in census', "s-9", 'cause "secret" : "" in census'), + (r'body {\"token\":\"xq7\"}', "xq7", r'body {\"token\":\"\"}'), + ('{"access_token":"at1","phase":"stage_engine_running"}', "at1", + '{"access_token":"","phase":"stage_engine_running"}'), + (r'{"token":"a\"b c","stage":"census"}', "b c", '{"token":"","stage":"census"}'), + (r'{\"token\":\"a1\\\"b2\",\"phase\":\"census\"}', "b2", + r'{\"token\":\"\",\"phase\":\"census\"}'), + ('"client_secret" = \'c c\' tail', "c c", '"client_secret" = \'\' tail'), + ('token: "no closing quote', "closing", 'token: ""'), + ("token = x9 then", "x9", "token = then"), + ("{token:k5}", "k5", "{token:}"), + ) + with mock.patch.dict(os.environ, {"HOME": "/home/runner"}): + for text, secret, saved in cases: + with self.subTest(text=text): + failure = self.journey["observer_failure"]("stage_engine_running", OSError(text)) + self.assertEqual(failure["message"], saved) + if secret: + self.assertNotIn(secret, failure["message"]) + def test_repair_control_failure_reaches_its_record(self): owner = self.journey["RepairRuntime"].__new__(self.journey["RepairRuntime"]) with tempfile.TemporaryDirectory() as temp: From 11b579ee828bd6e046555f7dd1f8066955fedc26 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 00:46:38 +0000 Subject: [PATCH 7/8] ci: finite observer failure fields from develop also go through redact() develop's finite observer saves only fixed messages, but its message and phase did not pass redact(). A message or phase added to that list later could carry a path or a secret value into the receipt. Both now pass redact() with the journey Home and data folder; every current fixed message is saved unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6Qn9RX7LDqrwiAfNC2F2K --- scripts/ci-installed-journeys.py | 10 +++++++--- scripts/ci-release-policy-test.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index 85d05804a..1e554e40a 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -202,12 +202,15 @@ def observer_failure(phase, error, roots=(), limit=480): }) -def finite_observer_failure(phase, error): +def finite_observer_failure(phase, error, roots=()): message = (error.args[0] if type(error) is RuntimeError and len(error.args) == 1 and type(error.args[0]) is str else None) if message not in FINITE_OBSERVER_MESSAGES: message = "observer detail redacted" - return {"phase": phase, "type": PUBLIC_ERROR_TYPES.get(type(error), "redacted"), "message": message[:480]} + # The fixed text still goes through redact(), so a message or phase added + # later cannot carry a path or a secret value into the receipt. + return {"phase": redact(phase, roots), "type": PUBLIC_ERROR_TYPES.get(type(error), "redacted"), + "message": redact(message, roots)[:480]} def failure_suffix(failure): @@ -688,7 +691,8 @@ def observe(self): except BaseException as error: # "census" with "Runtime left its unit" and runtime_exited true means # Runtime itself ended; other census messages name the failed check. - self.failure = {**finite_observer_failure(phase, error), "runtime_exited": self.runtime_exited()} + self.failure = {**finite_observer_failure(phase, error, (self.home, self.data)), + "runtime_exited": self.runtime_exited()} self.error = "finite_resource_observation_failed" private_json(self.control / "response.json", {"id": self.last_action + 1, "status": "failed"}) private_json(self.control / "stop.json", {"stop": True}) diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index b3c547c96..4b7912ea6 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -3233,6 +3233,23 @@ def test_finite_observer_saves_phase_type_and_only_public_messages(self): self.assertEqual(self.journey["private_json"](owner.control / "stop.json"), {"stop": True}) detail.__str__.assert_not_called() + def test_finite_observer_fixed_messages_and_phase_still_pass_redaction(self): + failure_of = self.journey["finite_observer_failure"] + # Every fixed message is readable after redact(), so it is saved as written. + for message in sorted(self.globals["FINITE_OBSERVER_MESSAGES"]): + with self.subTest(message=message): + self.assertEqual(failure_of("census", RuntimeError(message))["message"], message) + # A message or phase added later with a secret value or a path is still redacted. + added = 'finite journey control said {"token":"tq-41"} at /home/ci-person/x' + with mock.patch.dict(self.globals, {"FINITE_OBSERVER_MESSAGES": frozenset({added})}), \ + mock.patch.dict(os.environ, {"HOME": "/home/ci-person"}): + failure = failure_of('stage_{"token":"pq-17"}', RuntimeError(added)) + self.assertEqual(failure["message"], 'finite journey control said {"token":""} at ') + self.assertEqual(failure["phase"], 'stage_{"token":""}') + self.assertEqual(failure["type"], "RuntimeError") + for private in ("tq-41", "pq-17", "ci-person"): + self.assertNotIn(private, json.dumps(failure)) + def test_finite_observer_stage_phase_uses_only_validated_action_text(self): for action in ("engine_stopped", "private-action-Token1234567890"): with self.subTest(action=action), tempfile.TemporaryDirectory() as temp: From 04b621d0569543c6a00acddcd39307f7ca9958ae Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Sat, 17 Oct 2026 19:32:03 +0000 Subject: [PATCH 8/8] Use a platform-neutral app in Home link regression Use the installed System fixture for valid and invalid Home-link requests so the test exercises link authority on every host. Keep all assertions and production launch checks unchanged. Verified the exact Home-link test passes, reverting only the System token to Browser fails the intended 403-versus-200 assertion, and restoring the fixture passes again. Co-authored-by: Codex (cherry picked from commit 619c0961964aa2a5da938af5cba622349edb1382) --- .../src/api/gateway_tests/home_system.rs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs b/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs index a30bded9c..02d08fc68 100644 --- a/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs +++ b/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs @@ -11945,9 +11945,10 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { }) }; - let browser_token = launched(BROWSER_CAPSULE_ID); + // System is launchable on every host; Browser availability is platform-specific. + let system_token = launched(SYSTEM_CAPSULE_ID); for (app_token, app_id, app_title) in [ - (browser_token.clone(), BROWSER_CAPSULE_ID, "Browser"), + (system_token.clone(), SYSTEM_CAPSULE_ID, "System"), ( launched(HOME_CLI_CAPSULE_ID_FOR_TEST), "home-cli", @@ -11979,13 +11980,13 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { "foreign origin", "https://evil.example", home_token.clone(), - browser_token.clone(), + system_token.clone(), ), ( "an app frame calling directly", "null", home_token.clone(), - browser_token.clone(), + system_token.clone(), ), ( "app token from another Home session", @@ -11993,8 +11994,8 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { home_token.clone(), issue_home_projection_launch_token_with_context( dir.path(), - BROWSER_CAPSULE_ID, - BROWSER_CAPSULE_ID, + SYSTEM_CAPSULE_ID, + SYSTEM_CAPSULE_ID, &other_session, ) .unwrap(), @@ -12004,13 +12005,13 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { "http://localhost:61180", issue_home_launch_token_with_context(dir.path(), HOME_CAPSULE_ID, &other_session) .unwrap(), - browser_token.clone(), + system_token.clone(), ), ( "app token Home did not mint", "http://localhost:61180", home_token.clone(), - issue_home_launch_token_with_context(dir.path(), BROWSER_CAPSULE_ID, &context).unwrap(), + issue_home_launch_token_with_context(dir.path(), SYSTEM_CAPSULE_ID, &context).unwrap(), ), ( "Home GUI is not a link sender", @@ -12037,7 +12038,7 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { issue_home_projection_launch_token_with_context( dir.path(), "assistant", - BROWSER_CAPSULE_ID, + SYSTEM_CAPSULE_ID, &context, ) .unwrap(), @@ -12061,7 +12062,7 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { } let expired = - issue_expired_home_launch_token_with_context(dir.path(), BROWSER_CAPSULE_ID, &context) + issue_expired_home_launch_token_with_context(dir.path(), SYSTEM_CAPSULE_ID, &context) .unwrap(); let (status, body) = home_external_link_raw( &app, @@ -12088,7 +12089,7 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { &app, "http://localhost:61180", &home_token, - request(&browser_token, url), + request(&system_token, url), ) .await; assert_eq!(status, StatusCode::BAD_REQUEST, "{url:?}: {body}"); @@ -12099,13 +12100,13 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { &home_token, json!({ "schema": "elastos.home.external-link.request/v2", - "app_token": browser_token, + "app_token": system_token, "url": "https://example.com/", }), ) .await; assert_eq!(status, StatusCode::BAD_REQUEST); - let mut extra = request(&browser_token, "https://example.com/"); + let mut extra = request(&system_token, "https://example.com/"); extra["app_id"] = json!("wallet"); let (status, _) = home_external_link_raw(&app, "http://localhost:61180", &home_token, extra).await;