fix: make test suite and tools portable across default encodings and python -OO (#484)

.github/workflows/ci.yml

@@ -49,3 +49,8 @@ jobs:
 
       - name: Test
         run: pytest --cov=agentrust_trace --cov-report=term-missing
+
+      - name: Portability (non-UTF-8 locale and python -OO)
+        run: |
+          LC_ALL=C PYTHONUTF8=0 pytest
+          PYTHONOPTIMIZE=2 pytest tests/test_review_bar.py

conformance/tests/conftest.py

@@ -14,7 +14,7 @@
 
 
 def load_vector(filename):
-    return json.loads((VECTORS_DIR / filename).read_text())
+    return json.loads((VECTORS_DIR / filename).read_text(encoding="utf-8"))
 
 
 @pytest.fixture(name="load_vector")
@@ -23,7 +23,7 @@ def load_vector_fixture():
 
 
 def load_schema():
-    return json.loads((SCHEMAS_DIR / "trace-claim.json").read_text())
+    return json.loads((SCHEMAS_DIR / "trace-claim.json").read_text(encoding="utf-8"))
 
 
 def _b64url(b: bytes) -> str:

conformance/tests/test_anchor_vectors.py

@@ -11,7 +11,9 @@
 from trace_tests.modules import tr_anc
 
 VECTOR_DIR = Path(__file__).parent / "vectors" / "anchor-inclusion"
-VECTORS = [json.loads(p.read_text()) for p in sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json"))]
+VECTORS = [
+    json.loads(p.read_text(encoding="utf-8")) for p in sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json"))
+]
 
 
 @pytest.mark.level2

conformance/tests/test_manifest_ref_draft_vectors.py

@@ -19,7 +19,9 @@
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
 
 VECTOR_DIR = Path(__file__).parent / "vectors" / "manifest-ref-v0.3-draft"
-VECTORS = [json.loads(p.read_text()) for p in sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json"))]
+VECTORS = [
+    json.loads(p.read_text(encoding="utf-8")) for p in sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json"))
+]
 MUTANTS = (
     "trusts_asserted_result",
     "unresolved_is_pass",

conformance/tests/test_measurement_harness.py

@@ -10,8 +10,8 @@
 from `main()` left the whole suite green, which is precisely the property the script was
 written to detect, one level up.
 
-**Method.** Each test builds a synthetic checkout — the smallest tree with the shape the
-script requires — breaks one precondition, runs the script as a subprocess, and asserts
+**Method.** Each test builds a synthetic checkout, the smallest tree with the shape the
+script requires, breaks one precondition, runs the script as a subprocess, and asserts
 it refuses rather than reports. The synthetic tree carries one guarded check and one
 unguarded one, so the reporting path is exercised too and not only the aborts.
 
@@ -94,21 +94,22 @@ def _checkout(root: Path, *, module: str = MODULE, suite: str = SUITE) -> Path:
     (package / "modules").mkdir(parents=True)
     (root / "tests").mkdir()
     (root / "pyproject.toml").write_text(
-        '[tool.pytest.ini_options]\ntestpaths = ["tests"]\npythonpath = ["src"]\n'
+        '[tool.pytest.ini_options]\ntestpaths = ["tests"]\npythonpath = ["src"]\n',
+        encoding="utf-8",
     )
-    (package / "__init__.py").write_text("")
-    (package / "result.py").write_text(RESULT)
-    (package / "modules" / "__init__.py").write_text("")
-    (package / "modules" / "tr_fak.py").write_text(module)
-    (root / "tests" / "test_fake.py").write_text(suite)
+    (package / "__init__.py").write_text("", encoding="utf-8")
+    (package / "result.py").write_text(RESULT, encoding="utf-8")
+    (package / "modules" / "__init__.py").write_text("", encoding="utf-8")
+    (package / "modules" / "tr_fak.py").write_text(module, encoding="utf-8")
+    (root / "tests" / "test_fake.py").write_text(suite, encoding="utf-8")
     return root
 
 
 def _run(checkout: Path, *, pythonpath: str = "src") -> subprocess.CompletedProcess[str]:
     """Run the harness against *checkout*, as a person would.
 
     `PYTHONPATH` is relative and resolved by each subprocess against its own working
-    directory, which the script sets to the checkout — so "src" reaches that tree's
+    directory, which the script sets to the checkout, so "src" reaches that tree's
     package and stands in for the editable install a real checkout has.
     """
     return subprocess.run(
@@ -154,9 +155,9 @@ def test_the_module_is_restored_after_the_run(tmp_path: Path) -> None:
     corrupted the thing it was measuring."""
     checkout = _checkout(tmp_path)
     module = checkout / "src" / "trace_tests" / "modules" / "tr_fak.py"
-    before = module.read_text()
+    before = module.read_text(encoding="utf-8")
     _run(checkout)
-    assert module.read_text() == before
+    assert module.read_text(encoding="utf-8") == before
 
 
 # --- the guards, one broken precondition each ---------------------------------------
@@ -244,7 +245,7 @@ def test_a_suite_that_imports_another_tree_is_refused(tmp_path: Path) -> None:
 
     A stale editable install leaves the baseline green and every mutation unobserved:
     pytest never imports the file being rewritten, so every site reports that nothing
-    noticed — which reads exactly like a suite that guards nothing.
+    noticed, which reads exactly like a suite that guards nothing.
     """
     measured = _checkout(tmp_path / "measured")
     imported = _checkout(tmp_path / "imported")
@@ -257,7 +258,9 @@ def test_a_suite_that_imports_another_tree_is_refused(tmp_path: Path) -> None:
 def test_an_uninstallable_package_is_refused_rather_than_measured(tmp_path: Path) -> None:
     """The probe's own failure is a refusal too, not something to measure past."""
     checkout = _checkout(tmp_path)
-    (checkout / "src" / "trace_tests" / "__init__.py").write_text("raise ImportError('broken')\n")
+    (checkout / "src" / "trace_tests" / "__init__.py").write_text(
+        "raise ImportError('broken')\n", encoding="utf-8"
+    )
     result = _run(checkout)
     assert result.returncode != 0
     assert "could not import trace_tests" in result.stderr

conformance/tests/unit/test_cli.py

@@ -15,10 +15,10 @@
 
 @pytest.fixture
 def fresh_level0_path(tmp_path):
-    vector = json.loads((VECTORS_DIR / "valid_level0.json").read_text())
+    vector = json.loads((VECTORS_DIR / "valid_level0.json").read_text(encoding="utf-8"))
     vector["iat"] = int(time.time()) - 60
     p = tmp_path / "record.json"
-    p.write_text(json.dumps(vector))
+    p.write_text(json.dumps(vector), encoding="utf-8")
     return str(p)
 
 
@@ -52,19 +52,19 @@ def test_unsigned_record_level_0_reports_unverified(fresh_level0_path):
 
 
 def test_stale_record_fails(tmp_path):
-    vector = json.loads((VECTORS_DIR / "valid_level0.json").read_text())
+    vector = json.loads((VECTORS_DIR / "valid_level0.json").read_text(encoding="utf-8"))
     vector["iat"] = int(time.time()) - (25 * 3600)
     p = tmp_path / "stale.json"
-    p.write_text(json.dumps(vector))
+    p.write_text(json.dumps(vector), encoding="utf-8")
     result = CliRunner().invoke(main, ["verify", "--record", str(p), "--level", "0"])
     assert result.exit_code == 1, result.output
 
 
 def test_partial_cmcp_envelope_is_rejected(tmp_path):
-    vector = json.loads((VECTORS_DIR / "valid_cmcp_runtime.json").read_text())
+    vector = json.loads((VECTORS_DIR / "valid_cmcp_runtime.json").read_text(encoding="utf-8"))
     del vector["cmcp_version"]
     p = tmp_path / "partial.json"
-    p.write_text(json.dumps(vector))
+    p.write_text(json.dumps(vector), encoding="utf-8")
     result = CliRunner().invoke(main, ["verify", "--record", str(p), "--level", "0"])
     assert result.exit_code == 2, result.output
     assert "partial cmcp-runtime envelope" in result.output

conformance/tests/unit/test_loader.py

@@ -27,7 +27,7 @@
 
 def _write(tmp_path, data):
     p = tmp_path / "record.json"
-    p.write_text(json.dumps(data))
+    p.write_text(json.dumps(data), encoding="utf-8")
     return str(p)
 
 

conformance/tests/unit/test_runner.py

@@ -12,7 +12,7 @@
 
 
 def load_vector(filename: str) -> dict:
-    return json.loads((VECTORS_DIR / filename).read_text())
+    return json.loads((VECTORS_DIR / filename).read_text(encoding="utf-8"))
 
 
 @pytest.fixture

conformance/tests/vectors/manifest-ref-v0.3-draft/generate.py

@@ -144,4 +144,8 @@ def vectors() -> list[dict]:
 
 if __name__ == "__main__":
     for v in vectors():
-        (HERE / f"{v['name']}.json").write_text(json.dumps(v, indent=2, ensure_ascii=False) + "\n")
+        (HERE / f"{v['name']}.json").write_text(
+            json.dumps(v, indent=2, ensure_ascii=False) + "\n",
+            encoding="utf-8",
+            newline="\n",
+        )

examples/verifier-token-conformance/gen_corpus.py

@@ -2402,7 +2402,7 @@ def instance_preimage(tok):
 def legacy() -> None:
     token, reqs = base()
     for number, (stem, (req, desc)) in enumerate(sorted(LEGACY_MAP.items()), start=1):
-        old = json.loads((LEGACY / (stem + ".json")).read_text())
+        old = json.loads((LEGACY / (stem + ".json")).read_text(encoding="utf-8"))
         text = old["envelope_b64"]
         raw = base64.urlsafe_b64decode(text + "=" * (-len(text) % 4))
         code = old["expected"]

pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
     # `revocation.py` resolves the bundle schema's statement `$ref` from the packaged
     # copy through a `referencing.Registry`, so validation never fetches. `referencing`
     # already arrives through `jsonschema`; it is declared because it is imported.
-    "referencing>=0.28",
+    "referencing>=0.28.4",
     "rfc8785>=0.1.4",
 ]
 

requirements/dev.in

@@ -8,7 +8,7 @@ pydantic>=2.13.5
 jsonschema>=4.26.0
 rfc3986-validator>=0.1.1
 cryptography>=42.0
-referencing>=0.28
+referencing>=0.28.4
 rfc8785>=0.1.4
 # prototype extra: the experimental verifier-token reference and its tests
 cbor2>=5.6,<7

tests/test_acta_fixtures.py

@@ -23,7 +23,7 @@
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
 
 ACTA_DIR = Path(__file__).resolve().parents[1] / "examples" / "action-receipts" / "acta"
-EXPECTED = json.loads((ACTA_DIR / "expected.json").read_text())
+EXPECTED = json.loads((ACTA_DIR / "expected.json").read_text(encoding="utf-8"))
 FIXTURES = sorted(p.name for p in ACTA_DIR.glob("0*.json"))
 
 SIGNER_KEY = Ed25519PublicKey.from_public_bytes(
@@ -37,7 +37,7 @@ def _jcs(obj) -> bytes:
 
 
 def _load(name: str) -> dict:
-    return json.loads((ACTA_DIR / name).read_text())
+    return json.loads((ACTA_DIR / name).read_text(encoding="utf-8"))
 
 
 def _signature_verifies(envelope: dict) -> bool:
@@ -144,6 +144,8 @@ def test_key_mismatch_fixture_is_signed_by_the_committed_second_key():
     """03 is a real signature by the committed mismatched key, not random bytes."""
     env = _load("03-signature-key-mismatch.json")
     other = Ed25519PublicKey.from_public_bytes(
-        bytes.fromhex((ACTA_DIR / "mismatched-signer-public-key.txt").read_text().strip())
+        bytes.fromhex(
+            (ACTA_DIR / "mismatched-signer-public-key.txt").read_text(encoding="utf-8").strip()
+        )
     )
     other.verify(bytes.fromhex(env["signature"]["sig"]), _jcs(env["payload"]))

tests/test_adequacy_all_sets.py

@@ -30,7 +30,7 @@
 def _load(directory: str, outcome_of, codes_of, boundary_of=None) -> list[Vector]:
     out = []
     for path in sorted((EXAMPLES / directory).glob("*.json")):
-        expected = json.loads(path.read_text())["expected"]
+        expected = json.loads(path.read_text(encoding="utf-8"))["expected"]
         v = Vector(path.stem, outcome_of(expected), codes_of(expected))
         v.boundary = boundary_of(expected) if boundary_of else None
         out.append(v)

tests/test_azure_execution_binding.py

@@ -249,7 +249,7 @@ def _wcm():
     import wcm
 
     pem = (root / "amd-root.pem").read_bytes()
-    pinned = json.loads((root / "trust-root.json").read_text())["amd"]["pem_sha256"]
+    pinned = json.loads((root / "trust-root.json").read_text(encoding="utf-8"))["amd"]["pem_sha256"]
     assert hashlib.sha256(pem).hexdigest() == pinned
     trust = wcm.TrustStore()
     trust.add_root_pem(pem.decode())
@@ -258,8 +258,8 @@ def _wcm():
 
 def test_real_stock_boot_packet_replays_and_is_denied():
     wcm, trust = _wcm()
-    packet = json.loads((FIXTURES / "stock-relabel-packet.json").read_text())
-    meta = json.loads((FIXTURES / "stock-relabel-meta.json").read_text())
+    packet = json.loads((FIXTURES / "stock-relabel-packet.json").read_text(encoding="utf-8"))
+    meta = json.loads((FIXTURES / "stock-relabel-meta.json").read_text(encoding="utf-8"))
     ak = x.authenticate_ak(packet, trust)
     quote = x.parse_quote(base64.b64decode(packet["tpm_quote_b64"]))
     events = x.parse_uefi_log(base64.b64decode(packet["uefi_event_log_b64"]))
@@ -291,8 +291,8 @@ def test_real_stock_boot_packet_replays_and_is_denied():
 
 def test_legacy_pcr23_binding_accepts_the_same_unexecuted_agent():
     wcm, trust = _wcm()
-    packet = json.loads((FIXTURES / "stock-relabel-packet.json").read_text())
-    meta = json.loads((FIXTURES / "stock-relabel-meta.json").read_text())
+    packet = json.loads((FIXTURES / "stock-relabel-packet.json").read_text(encoding="utf-8"))
+    meta = json.loads((FIXTURES / "stock-relabel-meta.json").read_text(encoding="utf-8"))
     legacy = wcm.AzureSnpVtpmVerifier(trust).verify(
         packet["legacy_wcm_pcr23_bundle_b64"],
         expected_nonce=meta["challenge"],
@@ -309,12 +309,12 @@ def test_legacy_pcr23_binding_accepts_the_same_unexecuted_agent():
 
 
 def real(name):
-    packet = json.loads((FIXTURES / f"{name}-packet.json").read_text())
-    return packet, (FIXTURES / f"{name}.challenge").read_text().strip()
+    packet = json.loads((FIXTURES / f"{name}-packet.json").read_text(encoding="utf-8"))
+    return packet, (FIXTURES / f"{name}.challenge").read_text(encoding="utf-8").strip()
 
 
 def real_policy(**changes):
-    p = json.loads((FIXTURES / "execution-policy.json").read_text())
+    p = json.loads((FIXTURES / "execution-policy.json").read_text(encoding="utf-8"))
     policy = x.ExecutionPolicy(
         required_boot_application=p["required_boot_application"],
         approved_boot_applications=frozenset(p["approved_boot_applications"]),
@@ -329,7 +329,9 @@ def real_policy(**changes):
 def genoa_trust():
     wcm, trust = _wcm()
     pem = (FIXTURES / "ark-genoa.pem").read_bytes()
-    pinned = json.loads((FIXTURES / "execution-policy.json").read_text())["ark_genoa_sha256"]
+    pinned = json.loads((FIXTURES / "execution-policy.json").read_text(encoding="utf-8"))[
+        "ark_genoa_sha256"
+    ]
     assert hashlib.sha256(pem).hexdigest() == pinned
     trust.add_root_pem(pem.decode())
     return trust

tests/test_build_provenance_depth_doc.py

@@ -26,13 +26,13 @@
 
 ROOT = Path(__file__).resolve().parents[1]
 
-CANONICAL_SCHEMA = json.loads((ROOT / "schema" / "trace-claim.json").read_text())
+CANONICAL_SCHEMA = json.loads((ROOT / "schema" / "trace-claim.json").read_text(encoding="utf-8"))
 PACKAGED_SCHEMA = json.loads(
-    (ROOT / "src" / "agentrust_trace" / "schema" / "trace-v0.2.json").read_text()
+    (ROOT / "src" / "agentrust_trace" / "schema" / "trace-v0.2.json").read_text(encoding="utf-8")
 )
-NOTE = (ROOT / "docs" / "build-provenance-depth.md").read_text()
-SCHEMA_DOC = (ROOT / "docs" / "schema.md").read_text()
-TRUST_LEVELS_DOC = (ROOT / "docs" / "trust-levels.md").read_text()
+NOTE = (ROOT / "docs" / "build-provenance-depth.md").read_text(encoding="utf-8")
+SCHEMA_DOC = (ROOT / "docs" / "schema.md").read_text(encoding="utf-8")
+TRUST_LEVELS_DOC = (ROOT / "docs" / "trust-levels.md").read_text(encoding="utf-8")
 
 BUILD_PROVENANCE = CANONICAL_SCHEMA["properties"]["build_provenance"]
 

tests/test_build_provenance_depth_vectors.py

@@ -232,7 +232,7 @@ def verify(vector: dict[str, Any], depth: str, rules: Sequence[Rule] = RULES) ->
 
 
 def _load(path: Path) -> dict[str, Any]:
-    payload: dict[str, Any] = json.loads(path.read_text())
+    payload: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
     return payload
 
 

tests/test_conformance_causal.py

@@ -22,7 +22,7 @@
 from prototype import verifier_token as v
 from tests.test_conformance_corpus import CORPUS, NON_PORTABLE, REQUIREMENTS, load, outcome, vectors
 
-SOURCE = Path(v.__file__).read_text()
+SOURCE = Path(v.__file__).read_text(encoding="utf-8")
 _COUNTER = itertools.count()
 
 

tests/test_conformance_corpus.py

@@ -150,7 +150,7 @@
 
 
 def load(path: Path) -> dict[str, Any]:
-    return json.loads(path.read_text())
+    return json.loads(path.read_text(encoding="utf-8"))
 
 
 def unb64url(text: str) -> bytes:
@@ -245,7 +245,7 @@ def test_vector(path):
 
 def test_codes_are_the_published_list():
     codes = {c["code"] for c in load(CORPUS / "codes.json")["codes"]}
-    source = Path(v.__file__).read_text()
+    source = Path(v.__file__).read_text(encoding="utf-8")
     tree = ast.parse(source)
     raised = set()
     for node in ast.walk(tree):
@@ -384,7 +384,7 @@ def test_named_gateway_tests_exist():
     for requirement, row in coverage_from_vectors().items():
         for name in row["gateway_tests"]:
             file, function = name.split("::")
-            tree = ast.parse((ROOT / file).read_text())
+            tree = ast.parse((ROOT / file).read_text(encoding="utf-8"))
             names = {
                 node.name
                 for node in ast.walk(tree)
@@ -431,7 +431,7 @@ def write_coverage() -> None:
         "legacy": sorted(v["id"] for v in vectors() if v["id"].startswith("LEGACY-")),
         "divergences": DIVERGENCES,
     }
-    (CORPUS / "coverage.json").write_text(json.dumps(value, indent=1) + "\n")
+    (CORPUS / "coverage.json").write_text(json.dumps(value, indent=1) + "\n", encoding="utf-8")
 
 
 if __name__ == "__main__":

tests/test_delegation_completeness.py

@@ -386,7 +386,9 @@ def test_margins_have_not_thinned() -> None:
     current = {code: len(_margin(code)) for code in RULE_CODES}
 
     if not MARGINS_FILE.exists():
-        MARGINS_FILE.write_text(json.dumps(current, indent=2, sort_keys=True) + "\n")
+        MARGINS_FILE.write_text(
+            json.dumps(current, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
         pytest.skip(f"recorded initial margins to {MARGINS_FILE.name}; re-run to enforce")
 
     recorded: dict[str, int] = json.loads(MARGINS_FILE.read_text(encoding="utf-8"))