diff --git a/capsules/object-provider/Cargo.lock b/capsules/object-provider/Cargo.lock index d5114091d..914d1dc39 100644 --- a/capsules/object-provider/Cargo.lock +++ b/capsules/object-provider/Cargo.lock @@ -167,9 +167,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arc-swap" @@ -1190,7 +1187,7 @@ dependencies = [ "elastos-protected-content-provider-contracts", "elastos-protected-content-rights", "getrandom 0.2.17", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -1274,17 +1271,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -1589,7 +1575,7 @@ dependencies = [ "elastos-common", "elastos-compute", "libc", - "nix", + "nix 0.29.0", "serde", "serde_json", "thiserror 1.0.69", @@ -1689,7 +1675,7 @@ dependencies = [ "ghash", "hkdf", "hpke", - "nix", + "nix 0.29.0", "rand 0.10.2", "rand 0.9.5", "rand_chacha 0.3.1", @@ -1734,7 +1720,7 @@ dependencies = [ "elastos-protected-content-provider-contracts", "elastos-wallet-contract", "hex", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -3855,6 +3841,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -6053,6 +6051,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -7446,18 +7450,16 @@ dependencies = [ [[package]] name = "zip" -version = "2.4.2" +version = "9.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +checksum = "a954114b47a54df49f83d2722814e86b23d4bb1b54a1b90fb7d92b41619ddc22" dependencies = [ - "arbitrary", "crc32fast", - "crossbeam-utils", - "displaydoc", "flate2", "indexmap", "memchr", - "thiserror 2.0.20", + "nix 0.31.3", + "typed-path", ] [[package]] diff --git a/elastos/Cargo.lock b/elastos/Cargo.lock index 324256b60..eda3abe91 100644 --- a/elastos/Cargo.lock +++ b/elastos/Cargo.lock @@ -167,9 +167,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arc-swap" @@ -1197,7 +1194,7 @@ dependencies = [ "elastos-protected-content-provider-contracts", "elastos-protected-content-rights", "getrandom 0.2.17", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -1281,17 +1278,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -1598,7 +1584,7 @@ dependencies = [ "elastos-common", "elastos-compute", "libc", - "nix", + "nix 0.29.0", "serde", "serde_json", "tempfile", @@ -1706,7 +1692,7 @@ dependencies = [ "hkdf", "hpke", "k256", - "nix", + "nix 0.29.0", "rand 0.10.2", "rand 0.9.5", "rand_chacha 0.3.1", @@ -1765,7 +1751,7 @@ dependencies = [ "elastos-wallet-contract", "hex", "k256", - "nix", + "nix 0.29.0", "serde", "serde_json", "sha2 0.10.9", @@ -3926,6 +3912,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -6165,6 +6163,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -7704,18 +7708,16 @@ dependencies = [ [[package]] name = "zip" -version = "2.4.2" +version = "9.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +checksum = "a954114b47a54df49f83d2722814e86b23d4bb1b54a1b90fb7d92b41619ddc22" dependencies = [ - "arbitrary", "crc32fast", - "crossbeam-utils", - "displaydoc", "flate2", "indexmap", "memchr", - "thiserror 2.0.20", + "nix 0.31.3", + "typed-path", ] [[package]] diff --git a/elastos/crates/elastos-server/Cargo.toml b/elastos/crates/elastos-server/Cargo.toml index 56bf3b4ac..18fb4371a 100644 --- a/elastos/crates/elastos-server/Cargo.toml +++ b/elastos/crates/elastos-server/Cargo.toml @@ -104,7 +104,7 @@ tempfile = "3.10" # Archive extraction (for capsule artifacts) flate2 = "1.0" tar = "0.4" -zip = { version = "2.4.2", default-features = false, features = ["deflate-flate2", "flate2"] } +zip = { version = "9.0.4", default-features = false, features = ["deflate-flate2"] } libc = "0.2" # Built-in Carrier transport (P2P for updates, gossip messaging, DHT discovery). diff --git a/elastos/crates/elastos-server/src/api/gateway_tests/library.rs b/elastos/crates/elastos-server/src/api/gateway_tests/library.rs index d8477c0db..0253c229f 100644 --- a/elastos/crates/elastos-server/src/api/gateway_tests/library.rs +++ b/elastos/crates/elastos-server/src/api/gateway_tests/library.rs @@ -892,7 +892,7 @@ fn zip_text_files(bytes: &[u8]) -> std::collections::BTreeMap { if entry.is_dir() { continue; } - let name = entry.name().to_string(); + let name = entry.name().unwrap().into_owned(); let mut body = String::new(); entry.read_to_string(&mut body).unwrap(); files.insert(name, body); @@ -3870,6 +3870,203 @@ async fn test_library_provider_rejects_unsafe_zip_entries() { .contains("relative and safe")); } +/// Every path under `root` (files, folders and symlinks, not followed). +fn walk_paths(root: &std::path::Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + stack.push(path.clone()); + } + out.push(path); + } + } + out +} + +#[tokio::test] +async fn test_library_provider_extract_zip_refuses_absolute_dotdot_and_symlink_entries() { + use std::io::Write as _; + + let dir = tempfile::tempdir().unwrap(); + let app = gateway_router(library_test_state(dir.path()).await); + let authority = passkey_authority_with_name(dir.path(), Some("admin")); + let token = app_token_for_authority(dir.path(), LIBRARY_CAPSULE_ID, &authority); + let root = crate::auth::principal_localhost_root(&authority.principal_id); + let documents_uri = format!("{root}/Documents"); + let imports_uri = format!("{documents_uri}/Imports"); + + let (status, _) = post_library( + app.clone(), + &token, + "mkdir", + json!({ "parent_uri": root, "name": "Documents" }), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (status, _) = post_library( + app.clone(), + &token, + "mkdir", + json!({ "parent_uri": documents_uri, "name": "Imports" }), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let options = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + // Each archive holds one hostile entry. The whole-archive extract must + // refuse it and write nothing for it, inside or outside the Library. + // (archive name, entry kind, entry name, symlink target, expected message) + let cases: [(&str, &str, &str, &str, &str); 5] = [ + ( + "Absolute.zip", + "file", + "/absolute-escape.txt", + "", + "relative and safe", + ), + ( + "DotDot.zip", + "file", + "../dotdot-escape.txt", + "", + "relative and safe", + ), + ( + "Inner.zip", + "file", + "Nested/../inner-escape.txt", + "", + "relative and safe", + ), + ( + "Backslash.zip", + "file", + "..\\backslash-escape.txt", + "", + "invalid library object name", + ), + ( + "Symlink.zip", + "symlink", + "symlink-escape.txt", + "/etc/passwd", + "non-file", + ), + ]; + for (archive_name, kind, entry_name, link_target, expected) in cases { + let archive_bytes = { + let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + if kind == "symlink" { + writer + .add_symlink(entry_name, link_target, options) + .unwrap(); + } else { + writer.start_file(entry_name, options).unwrap(); + writer.write_all(b"escape").unwrap(); + } + writer.finish().unwrap().into_inner() + }; + let zip_uri = format!("{documents_uri}/{archive_name}"); + let (write_status, write) = post_library( + app.clone(), + &token, + "write", + json!({ + "uri": zip_uri, + "mime": "application/zip", + "data": base64::engine::general_purpose::STANDARD.encode(archive_bytes), + }), + ) + .await; + assert_eq!(write_status, StatusCode::OK, "{archive_name}"); + assert_eq!(write["status"], "ok", "{archive_name}: {write}"); + + let (extract_status, extracted) = post_library( + app.clone(), + &token, + "extract_archive", + json!({ "uri": zip_uri }), + ) + .await; + assert_eq!(extract_status, StatusCode::OK, "{archive_name}"); + assert_eq!( + extracted["status"], "error", + "{archive_name}: hostile entry {entry_name:?} was not refused: {extracted}" + ); + assert!( + extracted["message"].as_str().unwrap().contains(expected), + "{archive_name}: {extracted}" + ); + + if kind == "symlink" { + // The listing marks it blocked and a selected extract refuses it. + let (list_status, listed) = post_library( + app.clone(), + &token, + "archive_entries", + json!({ "uri": zip_uri }), + ) + .await; + assert_eq!(list_status, StatusCode::OK); + let rows = listed["data"]["entries"].as_array().unwrap(); + assert!( + rows.iter().any(|entry| entry["path"] == entry_name + && entry["kind"] == "blocked" + && entry["safety"]["reason"] + .as_str() + .unwrap() + .contains("non-file")), + "{listed}" + ); + let (select_status, selected) = post_library( + app.clone(), + &token, + "archive_extract_entries", + json!({ + "uri": zip_uri, + "destination_uri": imports_uri, + "entries": [entry_name], + }), + ) + .await; + assert_eq!(select_status, StatusCode::OK); + assert_eq!(selected["status"], "ok", "{selected}"); + assert_eq!( + selected["data"]["receipt"]["progress"]["blocked_entries"], + 1 + ); + assert!(selected["data"]["blocked"][0]["reason"] + .as_str() + .unwrap() + .contains("non-file")); + } + } + + // Nothing from any hostile entry exists anywhere under the data dir, and + // no symlink was created. + for path in walk_paths(dir.path()) { + let name = path.file_name().unwrap().to_string_lossy().to_string(); + assert!( + !name.contains("escape"), + "hostile entry was written: {}", + path.display() + ); + assert!( + !std::fs::symlink_metadata(&path) + .unwrap() + .file_type() + .is_symlink(), + "symlink was created: {}", + path.display() + ); + } +} + #[tokio::test] async fn test_library_gateway_reads_webspace_files_through_runtime_provider() { let dir = tempfile::tempdir().unwrap(); diff --git a/elastos/crates/elastos-server/src/library.rs b/elastos/crates/elastos-server/src/library.rs index 9e537d229..84c791d0a 100644 --- a/elastos/crates/elastos-server/src/library.rs +++ b/elastos/crates/elastos-server/src/library.rs @@ -4239,7 +4239,10 @@ fn preview_zip_archive_entry( let mut file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let raw_name = file.name().to_string(); + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))? + .into_owned(); let Ok(normalized) = normalized_archive_entry_path(Path::new(&raw_name)) else { continue; }; @@ -4457,7 +4460,10 @@ fn list_zip_archive_entries(bytes: Vec) -> anyhow::Result<(Vec, bool) let file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let raw_name = file.name().to_string(); + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))? + .into_owned(); let path = Path::new(&raw_name); let normalized = match normalized_archive_entry_path(path) { Ok(path) => path, @@ -4939,7 +4945,10 @@ fn collect_selected_zip_writes( let mut file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let raw_name = file.name().to_string(); + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))? + .into_owned(); let normalized = match normalized_archive_entry_path(Path::new(&raw_name)) { Ok(path) => path, Err(err) => { @@ -5429,8 +5438,10 @@ fn extract_zip_archive( let mut file = archive .by_index(index) .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; - let entry_path = file - .enclosed_name() + let raw_name = file + .name() + .map_err(|err| anyhow!("invalid ZIP archive entry: {err}"))?; + let entry_path = zip_entry_enclosed_path(&raw_name) .ok_or_else(|| anyhow!("library archive entry path must be relative and safe"))?; let entry_uri = archive_entry_uri(&destination.uri, &entry_path)?; if entry_uri == destination.uri { @@ -5461,6 +5472,29 @@ fn extract_zip_archive( Ok(()) } +/// The path a whole-archive ZIP extract writes for an entry name, or `None` +/// when the name holds a NUL byte, is absolute (a root or a drive prefix), or +/// climbs above the archive root. `archive_entry_uri` then refuses any `..` +/// that is left. This keeps the rule in the runtime: zip 9 `enclosed_name` +/// strips a leading `/` or drive prefix and folds `..` instead, which would +/// write entries that this extract refuses. +fn zip_entry_enclosed_path(name: &str) -> Option { + if name.contains('\0') { + return None; + } + let path = PathBuf::from(name); + let mut depth = 0usize; + for component in path.components() { + match component { + Component::Prefix(_) | Component::RootDir => return None, + Component::ParentDir => depth = depth.checked_sub(1)?, + Component::Normal(_) => depth += 1, + Component::CurDir => {} + } + } + Some(path) +} + fn archive_entry_uri(destination_uri: &str, path: &Path) -> anyhow::Result { let mut uri = destination_uri.trim_end_matches('/').to_string(); for component in path.components() { diff --git a/scripts/ci-installed-journeys.py b/scripts/ci-installed-journeys.py index ed7925052..1e554e40a 100644 --- a/scripts/ci-installed-journeys.py +++ b/scripts/ci-installed-journeys.py @@ -57,6 +57,103 @@ def private_json(path, value=None): return json.loads(encoded) +# Error type names that may leave the runner. Any other type name (a class a +# library or a test made) is saved as "redacted". +PUBLIC_ERROR_TYPES = { + RuntimeError: "RuntimeError", OSError: "OSError", FileNotFoundError: "FileNotFoundError", + PermissionError: "PermissionError", ProcessLookupError: "ProcessLookupError", + ValueError: "ValueError", KeyError: "KeyError", TypeError: "TypeError", + AttributeError: "AttributeError", UnicodeDecodeError: "UnicodeDecodeError", + json.JSONDecodeError: "JSONDecodeError", subprocess.SubprocessError: "SubprocessError", + subprocess.CalledProcessError: "CalledProcessError", subprocess.TimeoutExpired: "TimeoutExpired", +} + + +# A kept last path part must look like a harness file (request.json, +# memory.events). Any other last part can be a user, device or key name. +KEPT_PATH_SUFFIXES = {"json", "log", "txt", "events", "max", "current", "peak", "stat", "status", + "service", "sock", "lock", "pid", "bin", "data", "gguf", "car", "toml", "mjs", "py", "sh"} +SECRET_NAME = r"[A-Za-z0-9_.-]*(?:token|secret|passw(?:or)?d|passphrase|api[_-]?key|private[_-]?key|credential|cookie|session|signature|auth)[A-Za-z0-9_.-]*" +# A path runs until a mark that ends a path in an error message: a quote, +# a bracket, a comma, a semicolon, a pipe, a line end, or a colon followed by +# a space or the end ("open /a b/c: denied"). Spaces inside the path stay in +# it, so a folder name with a space is redacted whole ("/Volumes/Backup Disk"). +# A path that is not quoted or followed by such a mark takes the rest of the +# text with it: losing words is better than leaking a folder name. +PATH_STOP = "\\r\\n'\"`,;()\\[\\]{}<>|" +PATH_TAIL = rf"(?:[^{PATH_STOP}\s:]|:(?!\s|$)|[ \t]+(?=[^{PATH_STOP}\s:]|:(?!\s|$)))*" +PATH_START = "|".join(( + r"\bfile:/+", + r"(?/\\-])/(?![/\s])", +)) +# The user part of a URL (scheme://user:password@host) is a credential. +URL_USERINFO = re.compile(r"\b([A-Za-z][A-Za-z0-9+.-]*://)[^\s/?#'\"`<>]*@") +# The value of a secret-named key. The key may be bare, quoted or +# JSON-escaped (token=x, "token": "x", 'password': 'p', \"token\":\"x\"). +# A quoted value ends at the same quote with no backslash before it, so an +# escaped quote inside it stays redacted; with no closing quote the value +# takes the rest of the line. A bare value ends at a space, a quote, a comma, +# a semicolon or a closing bracket. +SECRET_VALUE = re.compile( + rf"""\b({SECRET_NAME})(\\*["']?\s*[=:]\s*)""" + r"""(?:(\\*["'])(?:.*?(?"), + (SECRET_VALUE, lambda found: found[1] + found[2] + ((found[3] or "") + "" + (found[3] or ""))), + # A DID names a key, so a person or a device: keep only its method. + (re.compile(r"\bdid:([a-z0-9]+):[^\s'\",;]+"), r"did:\1:"), + # Long hex, base64 or base58 runs: digests, keys, tokens, CIDs, unit and session ids. + (re.compile(r"(?"), +) + + +def redacted_path(path): + name = re.split(r"[/\\]", path.rstrip("/\\"))[-1] + found = re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\.([A-Za-z0-9]{1,8})", name) + return "/" + name if found and found[1].lower() in KEPT_PATH_SUFFIXES else "" + + +def redact(text, roots=()): + """Remove local paths, home folders and token-like values from text that + leaves the runner (receipts and CI log lines).""" + known = set() + for root in (*roots, os.environ.get("HOME")): + if root: + known.update({str(root), os.path.realpath(root)}) + try: + known.add(str(Path.home())) + except (KeyError, RuntimeError): + pass + text = URL_USERINFO.sub(r"\1@", text) + # Known roots first and longest first: a home or data folder may hold a space. + for root in sorted((root.rstrip("/") for root in known if len(root.rstrip("/")) > 1), key=len, reverse=True): + text = re.sub(re.escape(root) + rf"(?![^/\\{PATH_STOP}\s:]){PATH_TAIL}", lambda found: redacted_path(found[0]), text) + text = re.sub(rf"(?:{PATH_START}){PATH_TAIL}", lambda found: redacted_path(found[0]), text) + for pattern, replacement in REDACTIONS: + text = pattern.sub(replacement, text) + return text + + +def observer_failure(phase, error, roots=(), limit=480): + """Public cause of a repair control failure. The harness writes these + messages and no observer reads the private Runtime log; the text is still + redacted, because an OSError names local paths. The finite observer uses + the stricter fixed-message list below instead.""" + message = redact(" ".join(str(error).split()), roots) + if len(message) > limit: + message = message[:limit] + "..." + return {"phase": phase, "type": PUBLIC_ERROR_TYPES.get(type(error), "redacted"), "message": message} + + # Only fixed harness messages can leave the isolated Home. Other exception # details can contain paths, keys or Runtime text and use one redacted message. FINITE_OBSERVER_MESSAGES = frozenset({ @@ -105,20 +202,15 @@ def private_json(path, value=None): }) -def finite_observer_failure(phase, error): +def finite_observer_failure(phase, error, roots=()): message = (error.args[0] if type(error) is RuntimeError and len(error.args) == 1 and type(error.args[0]) is str else None) if message not in FINITE_OBSERVER_MESSAGES: message = "observer detail redacted" - public_types = { - RuntimeError: "RuntimeError", OSError: "OSError", FileNotFoundError: "FileNotFoundError", - PermissionError: "PermissionError", ProcessLookupError: "ProcessLookupError", - ValueError: "ValueError", KeyError: "KeyError", TypeError: "TypeError", - AttributeError: "AttributeError", UnicodeDecodeError: "UnicodeDecodeError", - json.JSONDecodeError: "JSONDecodeError", subprocess.SubprocessError: "SubprocessError", - subprocess.CalledProcessError: "CalledProcessError", subprocess.TimeoutExpired: "TimeoutExpired", - } - return {"phase": phase, "type": public_types.get(type(error), "redacted"), "message": message[:480]} + # The fixed text still goes through redact(), so a message or phase added + # later cannot carry a path or a secret value into the receipt. + return {"phase": redact(phase, roots), "type": PUBLIC_ERROR_TYPES.get(type(error), "redacted"), + "message": redact(message, roots)[:480]} def failure_suffix(failure): @@ -572,6 +664,12 @@ def action(self, request): "engine_pid": self.engine, "engine_start_ticks": self.known[self.engine]["start_ticks"]}) private_json(self.control / "response.json", {"id": self.last_action, "status": "passed", "engine_start_ticks": self.known[self.engine]["start_ticks"], "engine_pid": self.engine}) + def runtime_exited(self): + try: + return self.pid in self.fds and pidfd_exited(self.fds[self.pid]) + except OSError: + return None + def observe(self): phase = "census" try: @@ -591,12 +689,16 @@ def observe(self): phase = "deadline" raise RuntimeError("finite journey observer deadline") except BaseException as error: - self.failure = finite_observer_failure(phase, error) + # "census" with "Runtime left its unit" and runtime_exited true means + # Runtime itself ended; other census messages name the failed check. + self.failure = {**finite_observer_failure(phase, error, (self.home, self.data)), + "runtime_exited": self.runtime_exited()} self.error = "finite_resource_observation_failed" private_json(self.control / "response.json", {"id": self.last_action + 1, "status": "failed"}) private_json(self.control / "stop.json", {"stop": True}) def finish(self): + runtime_exited = self.runtime_exited() if self.pid is not None else None self.stop.set() if self.thread: self.thread.join(timeout=3) @@ -606,6 +708,8 @@ def finish(self): "actions": self.actions, "outside_runtime_unit": True} if self.failure: record["observer_failure"] = self.failure + if runtime_exited: + record["runtime_exited_before_cleanup"] = True phase = "unit_binding" try: if not self.launch_requested: @@ -1036,6 +1140,7 @@ def __init__(self, home, data, address, environment): self.stop = threading.Event() self.launch_lock = threading.Lock() self.error = None + self.failure = None try: self.launch() self.record = {"runtime_episodes": [self.identity()], "actions": []} @@ -1159,7 +1264,7 @@ def restart_get(self): return {**self.kept, "home_restarted": True} def observe(self): - previous = 0 + previous, phase = 0, "control_request" try: while not self.stop.wait(.05): try: @@ -1173,11 +1278,15 @@ def observe(self): action = request["action"] if action not in ("engine_running", "engine_stopped", "restart_get") or action in self.record["actions"]: raise RuntimeError("repair journey control action differs") + phase = "stage_" + action response = getattr(self, action)() + phase = "control_request" previous = request["id"] self.record["actions"].append(action) private_json(self.control / "response.json", {"id": previous, "status": "passed", **response}) - except BaseException: + except BaseException as error: + self.failure = observer_failure(phase, error, (self.home, self.data)) + self.record["control_failure"] = self.failure self.error = "installed_model_repair_control_failed" private_json(self.control / "response.json", {"id": previous + 1, "status": "failed"}) diff --git a/scripts/ci-model-package.mjs b/scripts/ci-model-package.mjs index 2d62dc955..6b977717d 100644 --- a/scripts/ci-model-package.mjs +++ b/scripts/ci-model-package.mjs @@ -41,7 +41,8 @@ export const rawCid = bytes => "b" + encode(Buffer.concat([Buffer.from([1, 0x55, // PROVENANCE.md is checked against a fixed charset before any download. const FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; const REPOSITORY = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/; -const APACHE_LICENSE_URL = "https://www.apache.org/licenses/LICENSE-2.0.txt"; +// The Apache License 2.0 text is vendored beside the fixtures; it has no network source. +const APACHE_LICENSE_PATH = "fixtures/licenses/Apache-2.0.txt"; export function loadFixture(path) { const fixture = JSON.parse(readFileSync(path, "utf8")); const manifest = fixture.capsule_manifest, model = manifest?.model_content, provenance = model?.provenance; @@ -57,7 +58,9 @@ export function loadFixture(path) { } const pinned = `https://huggingface.co/${provenance.quantized_repository}/resolve/${provenance.quantized_revision}/`; assert.equal(fixture.model.url, pinned + fixture.model.name, "fixture weights must come from the pinned quantized repository revision"); - assert([pinned + "LICENSE", APACHE_LICENSE_URL].includes(fixture.license.url), "fixture license must come from the pinned revision or apache.org"); + assert(fixture.license.url === undefined ? fixture.license.path === APACHE_LICENSE_PATH + : fixture.license.path === undefined && fixture.license.url === pinned + "LICENSE", + "fixture license must come from the pinned revision or the vendored Apache License text"); assert(["Q1_0", "Q4_K_M", "Q8_0"].includes(model.quantization), "fixture quantization is unsupported"); assert(model.license.spdx_id === "Apache-2.0" && provenance.base_license.spdx_id === "Apache-2.0", "fixture licenses must be Apache-2.0"); assert(/^[a-z0-9][a-z0-9._-]{0,127}$/.test(manifest.name), "fixture capsule name must be a plain identifier"); @@ -291,6 +294,15 @@ export async function download(source, path, get = fetch) { } } +// A vendored pin is checked in the repository before it is copied, and the +// copy is checked again. A changed vendored file stops the run. +export function copyVendored(source, path, base = join(root, "scripts")) { + if (existsSync(path)) return fileRecord(path, source.name, source); + fileRecord(join(base, source.path), source.name, source); + copyFileSync(join(base, source.path), path, constants.COPYFILE_EXCL); + return fileRecord(path, source.name, source); +} + // usage: produce // is a data dir prepared by scripts/seed-kubo-cache.sh (pinned recipe Kubo). async function produce(args) { @@ -302,8 +314,10 @@ async function produce(args) { assert.deepEqual(fileRecord(kubo, entrypoint.path), entrypoint, "Kubo binary matches its pinned recipe build"); const inputs = ownedDirectory(join(ownedDirectory(output), "inputs")), weights = join(inputs, fixture.model.name); assert.equal(dirname(weights), inputs, "fixture weights stay inside OUT/inputs"); + // The local license check runs first, so a changed copy stops before the model download. + if (fixture.license.path) copyVendored(fixture.license, join(inputs, "LICENSE")); + else await download(fixture.license, join(inputs, "LICENSE")); await download(fixture.model, weights); - await download(fixture.license, join(inputs, "LICENSE")); const repo = mkdtempSync(join(output, ".ipfs-repo-")); try { const { cid, bytes, packageDir } = producePackage({ inputs, output, publisher, fixture, add: dir => kuboAdd(kubo, repo, dir) }); diff --git a/scripts/ci-model-package.test.mjs b/scripts/ci-model-package.test.mjs index 231257669..50e31f8de 100644 --- a/scripts/ci-model-package.test.mjs +++ b/scripts/ci-model-package.test.mjs @@ -3,8 +3,8 @@ import test from "node:test"; import { createHash, createPublicKey, verify } from "node:crypto"; import { chmodSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { buildEntry, canonical, contentManifest, download, fileRecord, loadFixture, ownedDirectory, producePackage, rawCid, signedCatalog, SMOL_FIXTURE, verifyInstalledKubo, writeCatalogPayload } from "./ci-model-package.mjs"; +import { dirname, join } from "node:path"; +import { buildEntry, canonical, contentManifest, copyVendored, download, fileRecord, loadFixture, ownedDirectory, producePackage, rawCid, signedCatalog, SMOL_FIXTURE, verifyInstalledKubo, writeCatalogPayload } from "./ci-model-package.mjs"; const QWEN_FIXTURE = loadFixture(new URL("./pinned-qwen2.5-1.5b-fixture.json", import.meta.url)); @@ -64,7 +64,10 @@ test("fixture loading refuses escaping names, injected text, unpinned weights an refused(f => { provenance(f).quantized_repository = "Qwen/Qwen2.5\nSigned by someone else."; }, /owner\/name identifiers/); refused(f => { provenance(f).base_repository = "Qwen/../../etc"; }, /owner\/name identifiers/); refused(f => { f.model.url = f.model.url.replace(/resolve\/[0-9a-f]+\//, "resolve/main/"); }, /pinned quantized repository revision/); - refused(f => { f.license.url = "https://example.com/LICENSE"; }, /pinned revision or apache\.org/); + refused(f => { f.license.url = "https://example.com/LICENSE"; }, /pinned revision or the vendored Apache License text/); + refused(f => { f.license.url = "https://www.apache.org/licenses/LICENSE-2.0.txt"; }, /pinned revision or the vendored Apache License text/); + refused(f => { delete f.license.url; f.license.path = "../../outside/LICENSE"; }, /pinned revision or the vendored Apache License text/); + refused(f => { f.license.path = "fixtures/licenses/Apache-2.0.txt"; }, /pinned revision or the vendored Apache License text/); refused(f => { provenance(f).quantized_revision = "91cad511"; }, /full commit hashes/); refused(f => { f.capsule_manifest.description = "one\nline"; }, /one bounded line/); refused(f => { f.display_name = "Qwen 1.5B"; }, /display_name/); @@ -227,11 +230,26 @@ test("download refuses a pre-placed symlink partial and leaves its target untouc symlinkSync(victim, join(inputs, "LICENSE.partial")); let fetched = false; const get = async () => { fetched = true; return new Response("replaced"); }; - await assert.rejects(download(SMOL_FIXTURE.license, join(inputs, "LICENSE"), get), /EEXIST/); + await assert.rejects(download(QWEN_FIXTURE.license, join(inputs, "LICENSE"), get), /EEXIST/); assert.equal(readFileSync(victim, "utf8"), "keep"); assert.equal(fetched, false); }); +test("the SmolLM2 license is the vendored Apache text, copied only after its pin matches", t => { + assert.equal(SMOL_FIXTURE.license.url, undefined, "the SmolLM2 license has no network source"); + const root = temporary(t), base = join(root, "scripts"); + mkdirSync(join(root, "good")); mkdirSync(join(root, "tampered")); + const copied = copyVendored(SMOL_FIXTURE.license, join(root, "good/LICENSE")); + assert.deepEqual(copied, { path: "LICENSE", size: SMOL_FIXTURE.license.size, sha256: SMOL_FIXTURE.license.sha256 }); + mkdirSync(dirname(join(base, SMOL_FIXTURE.license.path)), { recursive: true }); + const text = readFileSync(join(root, "good/LICENSE"), "utf8"); + for (const [name, changed] of [["same size", text.replace("Apache License", "Apache Licanse")], ["appended", text + "extra grant\n"]]) { + writeFileSync(join(base, SMOL_FIXTURE.license.path), changed); + assert.throws(() => copyVendored(SMOL_FIXTURE.license, join(root, "tampered/LICENSE"), base), /pinned (checksum|size)/, name); + assert.throws(() => lstatSync(join(root, "tampered/LICENSE")), /ENOENT/, name); + } +}); + test("output directory refuses a symlinked ancestor", t => { const root = temporary(t), real = join(root, "real"); mkdirSync(real); diff --git a/scripts/ci-release-policy-test.py b/scripts/ci-release-policy-test.py index a576b012e..4b7912ea6 100644 --- a/scripts/ci-release-policy-test.py +++ b/scripts/ci-release-policy-test.py @@ -15,6 +15,7 @@ import tarfile import tempfile import textwrap +import time import unittest from unittest import mock @@ -2161,14 +2162,15 @@ def consumer_kubo(self, data, holds_root=False): (blocks / key[-3:-1]).mkdir() (blocks / key[-3:-1] / f"{key}.data").write_bytes(b"package root") - def execute(self, home, data, evidence, available=20 * 1024 ** 3, carrier=False, consumer_holds_root=False, memory_gib=0): + def execute(self, home, data, evidence, available=20 * 1024 ** 3, carrier=False, consumer_holds_root=False, memory_gib=0, + finite=None): journey = runpy.run_path(str(WORKFLOW.parents[2] / "scripts/ci-installed-journeys.py")) child = mock.Mock(pid=12345) child.poll.return_value = None response = mock.MagicMock() response.__enter__.return_value.status = 200 self.holder = mock.Mock(return_value=(mock.Mock(pid=23456), {"holder_did": "did:key:holder"})) - self.finite = mock.Mock() + self.finite = finite or mock.Mock() def node_journey(*args, **kwargs): if args[0][1] == "scripts/ci-model-package.mjs": @@ -3231,6 +3233,23 @@ def test_finite_observer_saves_phase_type_and_only_public_messages(self): self.assertEqual(self.journey["private_json"](owner.control / "stop.json"), {"stop": True}) detail.__str__.assert_not_called() + def test_finite_observer_fixed_messages_and_phase_still_pass_redaction(self): + failure_of = self.journey["finite_observer_failure"] + # Every fixed message is readable after redact(), so it is saved as written. + for message in sorted(self.globals["FINITE_OBSERVER_MESSAGES"]): + with self.subTest(message=message): + self.assertEqual(failure_of("census", RuntimeError(message))["message"], message) + # A message or phase added later with a secret value or a path is still redacted. + added = 'finite journey control said {"token":"tq-41"} at /home/ci-person/x' + with mock.patch.dict(self.globals, {"FINITE_OBSERVER_MESSAGES": frozenset({added})}), \ + mock.patch.dict(os.environ, {"HOME": "/home/ci-person"}): + failure = failure_of('stage_{"token":"pq-17"}', RuntimeError(added)) + self.assertEqual(failure["message"], 'finite journey control said {"token":""} at ') + self.assertEqual(failure["phase"], 'stage_{"token":""}') + self.assertEqual(failure["type"], "RuntimeError") + for private in ("tq-41", "pq-17", "ci-person"): + self.assertNotIn(private, json.dumps(failure)) + def test_finite_observer_stage_phase_uses_only_validated_action_text(self): for action in ("engine_stopped", "private-action-Token1234567890"): with self.subTest(action=action), tempfile.TemporaryDirectory() as temp: @@ -3268,6 +3287,206 @@ def test_finite_engine_stage_refuses_wrong_binding_interruption_and_deadline(sel self.assertEqual(owner.actions, []) self.assertFalse((owner.control / "response.json").exists()) + def test_finite_observer_failure_names_its_phase_type_message_and_runtime_exit(self): + cases = (("census", RuntimeError("finite journey Runtime left its unit"), True), + ("census", RuntimeError("finite journey child escaped its finite unit"), False), + ("stage_engine_running", RuntimeError("finite journey engine stage deadline"), False)) + for phase, error, exited in cases: + with self.subTest(phase=phase, error=str(error)), tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp)) + owner.started, owner.fds = time.monotonic(), {100: 1100} + owner.observe_once = mock.Mock(side_effect=None if phase.startswith("stage") else error) + owner.action = mock.Mock(side_effect=error) + if phase.startswith("stage"): + self.journey["private_json"](owner.control / "request.json", {"id": 1, "action": "engine_running"}) + self.journey["private_json"](owner.control / "runtime-stopped.json", {"reaped": True}) + with mock.patch.dict(self.globals, {"pidfd_exited": lambda fd: exited and fd == 1100}), \ + mock.patch.object(os, "close"): + owner.observe() + expected = {"phase": phase, "type": "RuntimeError", "message": str(error), "runtime_exited": exited} + self.assertEqual(owner.failure, expected) + self.assertEqual(owner.poll(), 1) + # The same cause reaches the CI log line that used to end the job without it. + with self.assertRaisesRegex(AssertionError, re.escape(f"exited before health ({phase}: RuntimeError: {error})")): + self.journey["wait_healthy"](owner, "127.0.0.1:1", "Runtime") + receipt = owner.finish() + self.assertEqual(receipt["status"], "failed") + self.assertEqual(receipt["observer_failure"], expected) + self.assertEqual(receipt.get("runtime_exited_before_cleanup"), True if exited else None) + + def test_observer_failure_text_is_bounded_and_single_line(self): + cause = self.journey["observer_failure"]("census", OSError("first line\nsecond " + "x" * 2000)) + self.assertEqual(cause["type"], "OSError") + self.assertNotIn("\n", cause["message"]) + self.assertTrue(cause["message"].startswith("first line second x")) + self.assertLessEqual(len(cause["message"]), 483) + + def test_finite_observer_failure_reaches_the_uploaded_installed_journey_receipt(self): + with tempfile.TemporaryDirectory() as temp: + owner, _ = self.finite_owner(Path(temp) / "observer") + owner.started, owner.fds, owner.launch_requested = time.monotonic(), {}, False + owner.observe_once = mock.Mock(side_effect=RuntimeError("finite journey Runtime left its unit")) + owner.observe() + owner.start = mock.Mock(return_value=owner) + case = InstalledJourneyTests() + home, data, evidence, _ = case.fixture(Path(temp) / "journey", "linux") + # The unit helper creates the private log before start() returns. + (home / "journey-runtime.private.log").write_text("private Runtime log line\n") + factory = mock.Mock(return_value=owner) + with self.assertRaisesRegex(AssertionError, re.escape( + "installed Runtime exited before health (census: RuntimeError: finite journey Runtime left its unit)")): + case.execute(home, data, evidence, memory_gib=4, finite=factory) + factory.assert_called_once() + record = json.loads((evidence / "installed-journeys.json").read_text()) + self.assertEqual(record["finite_memory"]["observer_failure"]["message"], "finite journey Runtime left its unit") + self.assertEqual(record["finite_memory"]["observer_failure"]["phase"], "census") + self.assertEqual(record["results"]["finite_memory_boundary"], "failed") + self.assertNotIn("private Runtime log line", (evidence / "installed-journeys.json").read_text()) + + def test_observer_failure_redacts_home_paths_and_tokens_before_the_record_is_saved(self): + hexdigest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + did = "z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK" + private = ["/home/ci-person", "ci-person", "observer root", "Application Support", "/opt/build-cache", + "eyJhbGciOiJFZERTQSJ9.payload-77", "s3cr3t-value", "hunter2pass", hexdigest, did, + "QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz", "q-secret-9"] + + def message(home, data): + return (f"[Errno 2] No such file or directory: '{data / 'finite-control/request.json'}' " + f"after {home}/model-inputs and /home/ci-person/.config/elastos/keys and " + f"/opt/build-cache/run-41/llama.bin and file:///home/ci-person/x; " + "Authorization: Bearer eyJhbGciOiJFZERTQSJ9.payload-77 token=s3cr3t-value " + f"password: hunter2pass digest {hexdigest} holder did:key:{did} " + "blob QWxhZGRpbjpvcGVuIHNlc2FtZQ9xYz body {\"token\": \"q-secret-9\"}") + + def assert_redacted(text, failure, finite=False): + for value in private: + self.assertNotIn(value, text) + if finite: + # The finite observer saves only its fixed harness messages. + self.assertEqual(failure["message"], "observer detail redacted") + return + self.assertIn("No such file or directory: '/request.json'", failure["message"]) + self.assertIn("did:key:", failure["message"]) + self.assertIn("Bearer ", failure["message"]) + + with tempfile.TemporaryDirectory() as temp, mock.patch.dict(os.environ, {"HOME": "/home/ci-person"}): + owner, _ = self.finite_owner(Path(temp) / "observer root") + owner.data = owner.home / "Library/Application Support/elastos" + owner.started, owner.fds, owner.launch_requested = time.monotonic(), {}, False + owner.observe_once = mock.Mock(side_effect=FileNotFoundError(message(owner.home, owner.data))) + owner.observe() + owner.start = mock.Mock(return_value=owner) + case = InstalledJourneyTests() + home, data, evidence, _ = case.fixture(Path(temp) / "journey", "linux") + (home / "journey-runtime.private.log").write_text("") + with self.assertRaisesRegex(AssertionError, "installed Runtime exited before health") as raised: + case.execute(home, data, evidence, memory_gib=4, finite=mock.Mock(return_value=owner)) + saved = (evidence / "installed-journeys.json").read_text() + failure = json.loads(saved)["finite_memory"]["observer_failure"] + self.assertEqual((failure["phase"], failure["type"], failure["runtime_exited"]), + ("census", "FileNotFoundError", False)) + assert_redacted(saved, failure, finite=True) + # The CI log line carries the same redacted cause. + assert_redacted(str(raised.exception), failure, finite=True) + + repair = self.journey["RepairRuntime"].__new__(self.journey["RepairRuntime"]) + repair.home = owner.home + repair.data = owner.data + repair.control, repair.record = Path(temp) / "repair-control", {"actions": []} + repair.control.mkdir(mode=0o700) + repair.stop = mock.Mock() + repair.stop.wait.return_value = False + self.journey["private_json"](repair.control / "request.json", {"id": 1, "action": "engine_running"}) + repair.engine_running = mock.Mock(side_effect=OSError(message(repair.home, repair.data))) + repair.observe() + self.assertEqual(repair.record["control_failure"]["phase"], "stage_engine_running") + assert_redacted(json.dumps(repair.record), repair.record["control_failure"]) + + def test_observer_failure_redacts_url_user_info_windows_network_paths_and_paths_with_spaces(self): + # Each case: the error text, the values that must not survive, and the + # cause that is saved. No case is under a known root, so only the + # general rules can catch it. + cases = ( + ("GET http://ci-user:hunter9@127.0.0.1:8090/api failed", ("ci-user", "hunter9"), + "GET http://@127.0.0.1:8090/api failed"), + ("proxy https://alice@proxy.corp.example:3128 refused", ("alice",), + "proxy https://@proxy.corp.example:3128 refused"), + ("fetch ftp://bob:pw-77@files.example/x (timeout)", ("bob", "pw-77"), + "fetch ftp://@files.example/x (timeout)"), + ("via socks5h://carol:p@ss@10.0.0.1:1080/", ("carol", "p@ss", "ss@"), + "via socks5h://@10.0.0.1:1080/"), + (r"open \\fileserver\share$\ci person\keys.json: denied", ("fileserver", "share$", "person"), + "open /keys.json: denied"), + ("open //fileserver/share/ci person/keys failed", ("fileserver", "share", "person"), "open "), + (r"\\?\C:\Users\ci person\AppData\x.json missing", ("Users", "person", "AppData"), ""), + (r"pipe \\.\pipe\elastos-ci-person busy", ("ci-person",), "pipe "), + (r"C:\Program Files\ElastOS Person\bin\elastos.exe: not found", ("Program", "Person"), + ": not found"), + ("C:/Users/ci person/AppData/Local/elastos/data failed", ("Users", "person"), ""), + ("cannot read /srv/ci person/Shared Keys/vault: Permission denied", ("srv", "person", "Shared", "Keys"), + "cannot read : Permission denied"), + ("No such file or directory: '/var/my folder/secret stuff/key.pem'", ("folder", "secret", "stuff"), + "No such file or directory: ''"), + ("error at /Volumes/Backup Disk/ci-person (exit 2)", ("Volumes", "Backup", "Disk", "ci-person"), + "error at (exit 2)"), + ) + with mock.patch.dict(os.environ, {"HOME": "/home/runner"}): + for text, private, saved in cases: + with self.subTest(text=text): + failure = self.journey["observer_failure"]("census", OSError(text)) + for value in private: + self.assertNotIn(value, failure["message"]) + self.assertEqual(failure["message"], saved) + # The rules keep what carries no person: a plain URL, a quoted + # command's options and a cause with no path. + for text in ("GET http://127.0.0.1:8090/api/health returned 503", "Expecting value: line 1 column 1 (char 0)", + "3/4 samples and/or a // b"): + with self.subTest(text=text): + self.assertEqual(self.journey["observer_failure"]("census", OSError(text))["message"], text) + + def test_observer_failure_redacts_secret_values_under_quoted_and_escaped_keys(self): + # Each case: the error text, the secret that must not survive, and the + # saved cause. The key and the names around it stay readable. + cases = ( + # The reviewer's case: one letter, so only the exact saved text proves it. + ('{"token":"x"}', None, '{"token":""}'), + ('{"api_key": "abc9"}', "abc9", '{"api_key": ""}'), + ("{'password': 'p4ss'}", "p4ss", "{'password': ''}"), + ('cause "secret" : "s-9 9" in census', "s-9", 'cause "secret" : "" in census'), + (r'body {\"token\":\"xq7\"}', "xq7", r'body {\"token\":\"\"}'), + ('{"access_token":"at1","phase":"stage_engine_running"}', "at1", + '{"access_token":"","phase":"stage_engine_running"}'), + (r'{"token":"a\"b c","stage":"census"}', "b c", '{"token":"","stage":"census"}'), + (r'{\"token\":\"a1\\\"b2\",\"phase\":\"census\"}', "b2", + r'{\"token\":\"\",\"phase\":\"census\"}'), + ('"client_secret" = \'c c\' tail', "c c", '"client_secret" = \'\' tail'), + ('token: "no closing quote', "closing", 'token: ""'), + ("token = x9 then", "x9", "token = then"), + ("{token:k5}", "k5", "{token:}"), + ) + with mock.patch.dict(os.environ, {"HOME": "/home/runner"}): + for text, secret, saved in cases: + with self.subTest(text=text): + failure = self.journey["observer_failure"]("stage_engine_running", OSError(text)) + self.assertEqual(failure["message"], saved) + if secret: + self.assertNotIn(secret, failure["message"]) + + def test_repair_control_failure_reaches_its_record(self): + owner = self.journey["RepairRuntime"].__new__(self.journey["RepairRuntime"]) + with tempfile.TemporaryDirectory() as temp: + owner.control, owner.record = Path(temp), {"actions": []} + owner.home, owner.data = Path(temp), Path(temp) / "data" + owner.control.chmod(0o700) + owner.stop = mock.Mock() + owner.stop.wait.return_value = False + self.journey["private_json"](owner.control / "request.json", {"id": 1, "action": "engine_running"}) + owner.engine_running = mock.Mock(side_effect=RuntimeError("repair journey engine deadline")) + owner.observe() + self.assertEqual(owner.error, "installed_model_repair_control_failed") + self.assertEqual(owner.record["control_failure"], {"phase": "stage_engine_running", "type": "RuntimeError", + "message": "repair journey engine deadline"}) + def test_finite_start_requires_actual_runtime_and_every_kernel_limit(self): for fault in (None, "launcher_pid", "wrong_parent", "unit_max", "swap", "tasks", "tighter_parent", "missing_max", "missing_current", "malformed_current"): @@ -3878,5 +4097,49 @@ def test_observer_refuses_unrelated_processes_and_foreign_engine_paths(self): self.assertEqual(list(observe({**rows, 3: (2, guard)}, 1, data)), []) +class PinnedModelProofLicenseTests(unittest.TestCase): + """The proof copies the vendored license text only after its pin matches; it never downloads it.""" + def proof(self, root, change=None): + tree = root / "tree" + (tree / "scripts/fixtures/licenses").mkdir(parents=True) + for name in ("pinned-model-consumer-proof.sh", "pinned-smollm2-fixture.json", "fixtures/licenses/Apache-2.0.txt"): + shutil.copy2(REPO / "scripts" / name, tree / "scripts" / name) + vendored = tree / "scripts/fixtures/licenses/Apache-2.0.txt" + if change: + vendored.write_bytes(change(vendored.read_bytes())) + tools = root / "tools" + tools.mkdir() + (tools / "curl").write_text("#!/bin/sh\ntouch \"$0.called\"\nexit 22\n") + (tools / "curl").chmod(0o700) + (root / "model.gguf").write_bytes(b"not the pinned model") + environment = {"PATH": f"{tools}:/usr/bin:/bin", "HOME": str(root), + "ELASTOS_PROOF_MODEL_PATH": str(root / "model.gguf")} + result = subprocess.run(["bash", str(tree / "scripts/pinned-model-consumer-proof.sh"), str(root / "proof"), + "--fetch", "--check-inputs"], env=environment, text=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60) + self.assertFalse((tools / "curl.called").exists(), "the license has no network source") + return result, root / "proof/inputs/LICENSE" + + def test_vendored_license_matches_its_pin_and_is_copied_before_model_checks(self): + with tempfile.TemporaryDirectory() as temp: + result, copied = self.proof(Path(temp)) + fixture = json.loads((REPO / "scripts/pinned-smollm2-fixture.json").read_text())["license"] + self.assertNotIn("url", fixture) + self.assertIn(f"vendored license verified: {fixture['size']} bytes sha256 {fixture['sha256']}", result.stdout) + self.assertEqual(hashlib.sha256(copied.read_bytes()).hexdigest(), fixture["sha256"]) + # The dummy model is refused after the license step; nothing else runs. + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("model size", result.stdout) + + def test_tampered_vendored_license_is_refused_before_copy_or_download(self): + for name, change in (("same size", lambda text: text.replace(b"Apache License", b"Apache Licanse", 1)), + ("appended", lambda text: text + b"extra grant\n"), ("empty", lambda text: b"")): + with self.subTest(change=name), tempfile.TemporaryDirectory() as temp: + result, copied = self.proof(Path(temp), change) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertRegex(result.stdout, r"vendored license (sha256|size) .* differs from pinned") + self.assertFalse(copied.exists() or copied.is_symlink()) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/fixtures/licenses/Apache-2.0.txt b/scripts/fixtures/licenses/Apache-2.0.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/scripts/fixtures/licenses/Apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/scripts/pinned-model-consumer-proof.sh b/scripts/pinned-model-consumer-proof.sh index 32d414e55..12ffd742a 100755 --- a/scripts/pinned-model-consumer-proof.sh +++ b/scripts/pinned-model-consumer-proof.sh @@ -30,7 +30,8 @@ Arguments: New or existing directory owned by this proof. It receives home/ (isolated HOME with the installed engine), kubo/, inputs/ (with --fetch) and proof/ (logs and receipt.json). - --fetch Download the pinned model and license into /inputs + --fetch Download the pinned model into /inputs and copy + the vendored license text there after checking its pin, when ELASTOS_PROOF_MODEL_PATH / _LICENSE_PATH are unset. --native-only Skip the cold Content proof (no Kubo or ipfs-provider). --check-inputs Verify pinned inputs and print the plan; run nothing else. @@ -82,7 +83,7 @@ fields = { "url": fixture["model"]["url"], "bytes": fixture["model"]["size"], "sha256": fixture["model"]["sha256"], - "license_url": fixture["license"]["url"], + "license_path": fixture["license"]["path"], "license_bytes": fixture["license"]["size"], "license_sha256": fixture["license"]["sha256"], "env_prefix": fixture["proof"]["env_prefix"], @@ -127,6 +128,18 @@ fetch_pinned() { chmod 0400 "$dest" } +# The license text lives in the repository beside its fixture pin and has no +# network source. Its bytes are checked before they are copied. +copy_vendored() { + local label="$1" source="$2" dest="$3" bytes="$4" sha="$5" + verify_pinned_file "vendored $label" "$source" "$bytes" "$sha" + if [[ ! -f "$dest" ]]; then + cp -- "$source" "${dest}.partial" + mv -f -- "${dest}.partial" "$dest" + fi + chmod 0400 "$dest" +} + PROOF_MODULE="api::capsule_inventory::preparation::tests::process_proof" # A cargo exit status of zero does not prove that the intended test ran: a @@ -269,14 +282,17 @@ mkdir -p "${PROOF_ROOT}/proof" "${PROOF_ROOT}/inputs" MODEL_PATH="${ELASTOS_PROOF_MODEL_PATH:-}" LICENSE_PATH="${ELASTOS_PROOF_LICENSE_PATH:-}" if [[ "$FETCH" == "1" ]]; then + # The local license check runs first, so a changed copy stops the run + # before the model download starts. + [[ -n "$LICENSE_PATH" ]] || { + LICENSE_PATH="${PROOF_ROOT}/inputs/LICENSE" + copy_vendored "license" "${ROOT}/scripts/$(pinned_field "$MODEL" license_path)" "$LICENSE_PATH" \ + "$(pinned_field "$MODEL" license_bytes)" "$(pinned_field "$MODEL" license_sha256)" + } [[ -n "$MODEL_PATH" ]] || { MODEL_PATH="${PROOF_ROOT}/inputs/$(pinned_field "$MODEL" file)" fetch_pinned "$(pinned_field "$MODEL" url)" "$MODEL_PATH" } - [[ -n "$LICENSE_PATH" ]] || { - LICENSE_PATH="${PROOF_ROOT}/inputs/LICENSE" - fetch_pinned "$(pinned_field "$MODEL" license_url)" "$LICENSE_PATH" - } fi [[ -n "$MODEL_PATH" && -n "$LICENSE_PATH" ]] \ || die "set ELASTOS_PROOF_MODEL_PATH and ELASTOS_PROOF_LICENSE_PATH, or pass --fetch" diff --git a/scripts/pinned-smollm2-fixture.json b/scripts/pinned-smollm2-fixture.json index cd16ff211..18d615574 100644 --- a/scripts/pinned-smollm2-fixture.json +++ b/scripts/pinned-smollm2-fixture.json @@ -9,7 +9,7 @@ }, "license": { "name": "LICENSE", - "url": "https://www.apache.org/licenses/LICENSE-2.0.txt", + "path": "fixtures/licenses/Apache-2.0.txt", "size": 11358, "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" },