From 67aa1c4ee282afeaacfcdb858d199ab6843b36fc Mon Sep 17 00:00:00 2001 From: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:00:22 -0500 Subject: [PATCH 1/3] =?UTF-8?q?docs/spec:=20close=20Go=20verifier=20findin?= =?UTF-8?q?gs=203,=2016,=2019=20in=20Anchor=20Format=20=C2=A71,=20verifier?= =?UTF-8?q?-compatibility=20README,=20and=20Acta=20crosswalk=20(#536)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - State the five language-independent canonical JSON rules in spec/registry-anchor-v1.md §1 (recursive Unicode code-point key sorting, ',' and ':' separators without whitespace, exact string and surrogate-pair escaping rules including \u007f for U+007F, integer 0 without minus sign, and no trailing newline). - Update examples/verifier-compatibility/README.md to reflect that Obligations 2 and 3 of #116 are now normative in spec/trace-v0.2.md §3.3 and documented in docs/verification.md ("Step 4: Check the EAT profile"), while Obligations 1 and 4 remain deferred. - Update the two stale v0.1 section 3.3.2 citations in docs/crosswalks/acta-decision-receipts.md to 3.3.3. - Add tests/test_registry_anchor_canonical_rules.py pinning all five rules across agentrust_trace.sign.anchor_bytes and trace_tests.inclusion.canonical_claim_bytes plus the documentation fixes. Closes #536 Refs #448 Signed-off-by: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/crosswalks/acta-decision-receipts.md | 4 +- examples/verifier-compatibility/README.md | 46 ++--- spec/registry-anchor-v1.md | 9 +- tests/test_registry_anchor_canonical_rules.py | 157 ++++++++++++++++++ 4 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 tests/test_registry_anchor_canonical_rules.py diff --git a/docs/crosswalks/acta-decision-receipts.md b/docs/crosswalks/acta-decision-receipts.md index aee25c65..869bc362 100644 --- a/docs/crosswalks/acta-decision-receipts.md +++ b/docs/crosswalks/acta-decision-receipts.md @@ -10,7 +10,7 @@ Acta is an IETF draft format for signed receipts that record whether an AI agent [Spec section 3.3.3](../../spec/trace-v0.2.md#333-action-receipts-for-embodied-workflows-informative) describes action receipts as evidence that sits below a Trust Record: a per-action signed statement, bound to a call or session, that a verifier checks independently of the core record. The [embodied-workflow fixtures](https://github.com/agentrust-io/trace-spec/blob/main/examples/action-receipts/README.md) show one profile of that pattern, for a physical controller (a robot arm, a safety monitor) signing an assertion about a physical action. -This cross-walk describes a second profile of the same pattern: an **Acta decision receipt**, evidencing a software decision (an AI agent's tool call, decided by a local policy gate before it runs) rather than a physical one. The verification pattern in 3.3.2 is deliberately domain-agnostic; the intent here is to show it composes with an existing, independently specified receipt format without a wire-format change to either side. +This cross-walk describes a second profile of the same pattern: an **Acta decision receipt**, evidencing a software decision (an AI agent's tool call, decided by a local policy gate before it runs) rather than a physical one. The verification pattern in 3.3.3 is deliberately domain-agnostic; the intent here is to show it composes with an existing, independently specified receipt format without a wire-format change to either side. This follows the acceptance bar raised on [trace-spec#97](https://github.com/agentrust-io/trace-spec/issues/97): carry Acta as external evidence by digest/chain head and issuer/key metadata rather than embedding a second receipt schema into the Trust Record; state exactly which Acta fields satisfy the action-receipt verifier obligations; and keep the physical/software boundary explicit. @@ -57,7 +57,7 @@ Field names on the left are exact TRACE terms as used in [spec section 3.3.3](.. | `receipt.issuer_key_id` | `signature.kid` | Resolved the same way 3.3.3 describes: through a pinned, manifest-bound, or otherwise trusted key set, not carried in the receipt as a public key. | | `receipt.linked_call_id` / `session_id` | `payload.session_id` | Acta s3.1 binds at session level; `session_id` is OPTIONAL, and when present MUST NOT contain PII or be correlatable across sessions unless the operator explicitly configures session binding. Acta has no per-call identifier field; a TRACE-composing deployment gets call-level binding by carrying the tool input's digest in `payload_digest` (below) or by a deployment extension field, and should say which. | | `receipt.evidence_type` | `payload.type` | Namespaced receipt type (Acta s2.2), here `protectmcp:decision`. | -| `receipt.evidence_hash` | `payload.payload_digest` | OPTIONAL common field (Acta s2.2): `{hash, size, preview?}` over associated data (tool input/output) too large to embed. Serves exactly the external-evidence-by-digest role 3.3.2 describes. | +| `receipt.evidence_hash` | `payload.payload_digest` | OPTIONAL common field (Acta s2.2): `{hash, size, preview?}` over associated data (tool input/output) too large to embed. Serves exactly the external-evidence-by-digest role 3.3.3 describes. | | `receipt.previous_receipt_hash` | `payload.previousReceiptHash` | Bare lowercase hex over the predecessor's full envelope including its signature (Acta s5.7). Note the deliberate difference from formats that hash payload only: re-signing an identical payload yields a distinct chain link. | | `receipt.decision` | `payload.decision` (+ `payload.reason`) | `reason` (OPTIONAL) names the machine-readable cause, e.g. `policy_block`, `tier_insufficient`, beyond the bare outcome. | | `receipt.signature` | `signature.sig` (with `signature.alg`) | `EdDSA` is mandatory-to-implement; `ES256` optional, `ML-DSA-65` recommended for new deployments (Acta s5.8). | diff --git a/examples/verifier-compatibility/README.md b/examples/verifier-compatibility/README.md index 4fe0ae7e..55bdb13c 100644 --- a/examples/verifier-compatibility/README.md +++ b/examples/verifier-compatibility/README.md @@ -1,20 +1,26 @@ -# Verifier-compatibility vectors (proposed, not accepted) +# Verifier-compatibility vectors -> These fixtures encode the obligations proposed in -> [agentrust-io/trace-spec#116](https://github.com/agentrust-io/trace-spec/issues/116). -> **No normative text for them has been accepted.** Do not read a passing vector here -> as a conformance requirement. +> These fixtures encode Obligations 2 and 3 of +> [agentrust-io/trace-spec#116](https://github.com/agentrust-io/trace-spec/issues/116), +> which are normative in `spec/trace-v0.2.md` §3.3 and documented in +> `docs/verification.md` ("Step 4: Check the EAT profile"). Obligations 1 (emitter +> version declarations) and 4 (downgrade disclosure) remain deferred. Compliance evidence is verified years after it is issued, so the artifact routinely -outlives the verifier build that reads it. Nothing in `docs/verification.md` currently -says what a verifier must do about version skew, which leaves two failure modes open: -a revision silently orphans existing evidence, or a verifier "helpfully" accepts an -artifact under semantics it does not implement. - -The proposal is four obligations: records carry an explicit profile, a verifier -declares the set it supports and refuses anything outside it, the verification -statement names the profile it ran under, and any fallback is disclosed rather than -silent. +outlives the verifier build that reads it. `spec/trace-v0.2.md` §3.3 and +`docs/verification.md` ("Step 4: Check the EAT profile") address version skew by +requiring a nonempty `accepted_profiles` set and recording both the verified `profile` +and the complete `accepted_profiles` set in the verification result. Without those +checks, two failure modes stay open: a revision silently orphans existing evidence, or +a verifier "helpfully" accepts an artifact under semantics it does not implement. + +#116 proposed four obligations: records carry an explicit profile (Obligation 1, +deferred for standalone emitter version declarations beyond `eat_profile`), a verifier +declares the nonempty `accepted_profiles` set it supports and refuses anything outside +it (Obligation 2, normative in `spec/trace-v0.2.md` §3.3), the verification statement +records both the verified `profile` and the complete `accepted_profiles` set +(Obligation 3, normative in `spec/trace-v0.2.md` §3.3), and any fallback is disclosed +rather than silent (Obligation 4, deferred). ## The vectors @@ -85,12 +91,12 @@ reports one generic refusal for all of them conforms, and two controls hold that `test_a_generic_refusal_is_separated_by_nothing` in the separation module. Both fail the day an assertion on the refusal's cause comes back. -This was settled on review, and the set was stricter than its text until it was. The draft -normative text for #116, held for a maintainer to carry and not part of this change, says -a verifier SHOULD report refusal-for-an-unimplemented-profile distinguishably from a -verification failure, which is coarser than a rule name and is a SHOULD, and it declines -to mandate any field name. Asserting the label was this set asking more of a foreign -implementation than the text it encodes. +This was settled on review, and the set was stricter than its text until it was. The +proposal for #116 said a verifier SHOULD report refusal-for-an-unimplemented-profile +distinguishably from a verification failure, which is coarser than a rule name, and +neither #116 nor `spec/trace-v0.2.md` §3.3 mandates any refusal field name. Asserting +the label was this set asking more of a foreign implementation than the specification +requires. The label stays in the JSON because it tells a reader what each vector is for. This library's own refusal messages are pinned separately, in diff --git a/spec/registry-anchor-v1.md b/spec/registry-anchor-v1.md index d030ef89..2d5a1686 100644 --- a/spec/registry-anchor-v1.md +++ b/spec/registry-anchor-v1.md @@ -32,7 +32,7 @@ The two agree on records whose strings and keys are pure ASCII and whose numbers - **Non-ASCII strings.** JCS emits the character; §1 emits a `\uXXXX` escape. - **Non-integer numbers.** JCS applies ECMAScript number serialization. §1 excludes non-integer numbers from this profile for that reason. -- **Integers outside the safe-integer range.** JCS converts a number through an IEEE 754 double, so `9007199254740992` and `9007199254740993` become the same bytes; §1 serializes the digits as written, so they stay distinct. Two implementations of §1 do not even agree with each other here: Python's `json.dumps` writes the exact digits, and a JavaScript implementation of the same four rules goes through `JSON.stringify` and emits one value for both, which is a rug-pull this profile exists to catch going undetected. §1 excludes the range for that reason. +- **Integers outside the safe-integer range.** JCS converts a number through an IEEE 754 double, so `9007199254740992` and `9007199254740993` become the same bytes; §1 serializes the digits as written, so they stay distinct. Two implementations of §1 do not even agree with each other here: Python's `json.dumps` writes the exact digits, and a JavaScript implementation of the same five rules goes through `JSON.stringify` and emits one value for both, which is a rug-pull this profile exists to catch going undetected. §1 excludes the range for that reason. - **Key order.** JCS sorts by UTF-16 code unit (RFC 8785 §3.2.3); §1 sorts by Unicode code point, which is what Python's `sort_keys=True` does. The two agree across the Basic Multilingual Plane and disagree once a key contains a supplementary-plane character, because surrogates occupy U+D800 to U+DFFF. Do not "simplify" a verifier by reusing the signing canonicalizer at the leaf. If you do, write a test that anchors a record containing a non-ASCII character and verifies it, which is the test that catches this. @@ -45,9 +45,10 @@ The unit of anchoring is the **complete signed claim object, signature included* `canonical_claim_bytes` is the canonical JSON serialization of the claim object: -- Object keys sorted lexicographically by Unicode code point, recursively. +- Object keys sorted lexicographically by Unicode code point, recursively (UTF-16 surrogate pairs are not used for key comparison; see §0). - Separators `","` and `":"`, no whitespace. -- Non-ASCII characters escaped (`ensure_ascii`); output encoded as ASCII bytes. +- String escaping: `"` (`U+0022`), `\` (`U+005C`), and the five control characters `U+0008`, `U+0009`, `U+000A`, `U+000C`, and `U+000D` use the two-character escapes `\"`, `\\`, `\b`, `\t`, `\n`, `\f`, and `\r`; `/` (`U+002F`) is not escaped; all other control characters (`U+0000` to `U+001F`) and all BMP characters from `U+007F` to `U+FFFF` (including `U+007F`, which is escaped as `\u007f`) use lowercase four-hex-digit `\uXXXX` escapes; supplementary-plane characters (`U+10000` to `U+10FFFF`) use lowercase UTF-16 surrogate pairs (a high surrogate in `U+D800` to `U+DBFF` followed by a low surrogate in `U+DC00` to `U+DFFF`, e.g. `\ud8xx\udcxx`); output is encoded as ASCII bytes. +- Integer zero is written `0` (never `-0`). - No trailing newline. In Python this is exactly: @@ -56,7 +57,7 @@ In Python this is exactly: json.dumps(claim, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii") ``` -Claims MUST be JSON objects. TRACE Trust Records contain only strings, integers, booleans, nulls, arrays, and objects. Claims containing non-integer numbers are outside this profile, because cross-language float serialization is not canonical, and claims containing an integer outside -9007199254740991 to 9007199254740991 are outside it for the same reason: an implementation of the four rules above in a language whose only number type is the IEEE 754 double writes one value for two distinct integers. TRACE v0.2 §3.2.2 already holds a Trust Record to that range, so a schema-valid record cannot reach this; a claim that is not a Trust Record can, and this is where it is excluded. +Claims MUST be JSON objects. TRACE Trust Records contain only strings, integers, booleans, nulls, arrays, and objects. Claims containing non-integer numbers are outside this profile, because cross-language float serialization is not canonical, and claims containing an integer outside -9007199254740991 to 9007199254740991 are outside it for the same reason: an implementation of the five rules above in a language whose only number type is the IEEE 754 double writes one value for two distinct integers. TRACE v0.2 §3.2.2 already holds a Trust Record to that range, so a schema-valid record cannot reach this; a claim that is not a Trust Record can, and this is where it is excluded. ## 2. Leaf hash diff --git a/tests/test_registry_anchor_canonical_rules.py b/tests/test_registry_anchor_canonical_rules.py new file mode 100644 index 00000000..442f519b --- /dev/null +++ b/tests/test_registry_anchor_canonical_rules.py @@ -0,0 +1,157 @@ +"""Tests for Registry Anchor Format v1 §1 canonical serialization rules (#536).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from agentrust_trace.sign import anchor_bytes +from trace_tests.inclusion import canonical_claim_bytes + +REPO_ROOT = Path(__file__).resolve().parents[1] +SERIALIZERS = (anchor_bytes, canonical_claim_bytes) + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +def test_rule_1_recursive_code_point_key_sorting(serialize) -> None: + """Object keys are sorted lexicographically by Unicode code point, recursively.""" + # U+FFFD (65533) < U+10000 (65536) by code point, whereas in UTF-16 code units + # U+10000 (\uD800\uDC00) precedes U+FFFD (\uFFFD). + claim = { + "z": {"b": 1, "a": 2}, + "\U00010000": "supplementary", + "\ufffd": "bmp", + "a": 0, + } + assert serialize(claim) == ( + b'{"a":0,"z":{"a":2,"b":1},"\\ufffd":"bmp","\\ud800\\udc00":"supplementary"}' + ) + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +def test_rule_2_separators_and_rule_5_no_trailing_newline(serialize) -> None: + """Separators are ',' and ':', with no whitespace and no trailing newline.""" + out = serialize({"b": [1, 2, 3], "a": {"k": "v"}}) + assert out == b'{"a":{"k":"v"},"b":[1,2,3]}' + assert not out.endswith(b"\n") + assert b" " not in out + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +@pytest.mark.parametrize( + ("char", "escaped"), + [ + ('"', b'\\"'), + ("\\", b"\\\\"), + ("\b", b"\\b"), + ("\t", b"\\t"), + ("\n", b"\\n"), + ("\f", b"\\f"), + ("\r", b"\\r"), + ("/", b"/"), + ], +) +def test_rule_3_short_escapes_and_unescaped_slash( + serialize, char: str, escaped: bytes +) -> None: + """Mandatory two-character escapes and unescaped '/' (U+002F).""" + assert serialize({"s": char}) == b'{"s":"' + escaped + b'"}' + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +def test_rule_3_remaining_control_characters_use_lowercase_hex(serialize) -> None: + """Control characters U+0000..U+001F other than \\b, \\t, \\n, \\f, \\r use \\u00xx.""" + short = {0x08, 0x09, 0x0A, 0x0C, 0x0D} + for cp in range(0x20): + if cp in short: + continue + expected = f'\\u{cp:04x}'.encode("ascii") + assert serialize({"s": chr(cp)}) == b'{"s":"' + expected + b'"}' + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +def test_rule_3_del_and_bmp_above_007f_use_lowercase_hex( + serialize, +) -> None: + """U+007F (DELETE) and U+0080..U+FFFF use lowercase \\uXXXX escapes.""" + for cp, hex_escape in [ + (0x007F, b"\\u007f"), + (0x0080, b"\\u0080"), + (0x00E9, b"\\u00e9"), + (0x20AC, b"\\u20ac"), + (0xFFFD, b"\\ufffd"), + (0xFFFF, b"\\uffff"), + ]: + assert serialize({"s": chr(cp)}) == b'{"s":"' + hex_escape + b'"}' + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +def test_rule_3_supplementary_plane_uses_lowercase_surrogate_pairs(serialize) -> None: + """Supplementary-plane code points U+10000..U+10FFFF use lowercase surrogate pairs.""" + assert serialize({"s": "\U0001f600"}) == b'{"s":"\\ud83d\\ude00"}' + assert serialize({"s": "\U00010000"}) == b'{"s":"\\ud800\\udc00"}' + assert serialize({"s": "\U0010ffff"}) == b'{"s":"\\udbff\\udfff"}' + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +def test_rule_3_escaping_applies_to_object_keys_as_well_as_values(serialize) -> None: + """Object member names apply the same escaping rules as string values.""" + claim = { + 'a"b\\c/d\n\x00\x7f\u00e9\U0001f600': 1, + } + assert ( + serialize(claim) + == b'{"a\\"b\\\\c/d\\n\\u0000\\u007f\\u00e9\\ud83d\\ude00":1}' + ) + + +@pytest.mark.parametrize("serialize", SERIALIZERS) +def test_rule_4_integer_zero_is_written_without_minus_sign(serialize) -> None: + """Integer zero (-0) is serialized as 0, never -0.""" + assert serialize({"n": -0}) == b'{"n":0}' + + +def test_registry_anchor_spec_documents_all_five_canonical_rules() -> None: + """spec/registry-anchor-v1.md §1 states all five language-independent rules.""" + doc = (REPO_ROOT / "spec" / "registry-anchor-v1.md").read_text(encoding="utf-8") + section_1 = doc.split("## 1. Canonical claim bytes")[1].split("## 2. Leaf hash")[0] + + for required_fragment in ( + "Object keys sorted lexicographically by Unicode code point, recursively", + 'Separators `","` and `":"`, no whitespace.', + r'`\"`, `\\`, `\b`, `\t`, `\n`, `\f`, and `\r`', + "`/` (`U+002F`) is not escaped", + "(`U+0000` to `U+001F`)", + ( + "from `U+007F` to `U+FFFF` (including `U+007F`, which is escaped as " + "`\\u007f`) use lowercase four-hex-digit `\\uXXXX` escapes" + ), + ( + "(`U+10000` to `U+10FFFF`) use lowercase UTF-16 surrogate pairs " + "(a high surrogate in `U+D800` to `U+DBFF` followed by a low surrogate in " + "`U+DC00` to `U+DFFF`, e.g. `\\ud8xx\\udcxx`)" + ), + "Integer zero is written `0` (never `-0`).", + "No trailing newline.", + ): + assert required_fragment in section_1, ( + f"spec/registry-anchor-v1.md §1 is missing {required_fragment!r}" + ) + + +def test_verifier_compatibility_readme_and_acta_crosswalk_findings_closed() -> None: + """#448 items 3 and 16 remain closed in the documentation.""" + readme = ( + REPO_ROOT / "examples" / "verifier-compatibility" / "README.md" + ).read_text(encoding="utf-8") + assert "No normative text for them has been accepted" not in readme + assert "Nothing in `docs/verification.md` currently" not in readme + assert "`spec/trace-v0.2.md` §3.3" in readme + assert "`docs/verification.md`" in readme + + acta = ( + REPO_ROOT / "docs" / "crosswalks" / "acta-decision-receipts.md" + ).read_text(encoding="utf-8") + assert "3.3.2" not in acta + assert "3.3.3" in acta From 92ef2fdb25385e4c8e650a431e3602575a607993 Mon Sep 17 00:00:00 2001 From: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:49:42 -0500 Subject: [PATCH 2/3] test: load conformance inclusion module via spec_from_file_location (#536) The root CI workflow installs only the root agentrust-trace package and does not place conformance/src on PYTHONPATH. Load conformance/src/trace_tests/inclusion.py via importlib.util.spec_from_file_location so test_registry_anchor_canonical_rules.py collects and runs in the root CI environment. Signed-off-by: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_registry_anchor_canonical_rules.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_registry_anchor_canonical_rules.py b/tests/test_registry_anchor_canonical_rules.py index 442f519b..cd12aa6f 100644 --- a/tests/test_registry_anchor_canonical_rules.py +++ b/tests/test_registry_anchor_canonical_rules.py @@ -2,14 +2,22 @@ from __future__ import annotations +import importlib.util from pathlib import Path import pytest from agentrust_trace.sign import anchor_bytes -from trace_tests.inclusion import canonical_claim_bytes REPO_ROOT = Path(__file__).resolve().parents[1] +_INCLUSION_SPEC = importlib.util.spec_from_file_location( + "inclusion", REPO_ROOT / "conformance" / "src" / "trace_tests" / "inclusion.py" +) +assert _INCLUSION_SPEC is not None and _INCLUSION_SPEC.loader is not None +_INCLUSION_MODULE = importlib.util.module_from_spec(_INCLUSION_SPEC) +_INCLUSION_SPEC.loader.exec_module(_INCLUSION_MODULE) +canonical_claim_bytes = _INCLUSION_MODULE.canonical_claim_bytes + SERIALIZERS = (anchor_bytes, canonical_claim_bytes) From 563a8c3ffc8ac276f5f1c9cf70013c8176a1c6d7 Mon Sep 17 00:00:00 2001 From: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:26:34 -0500 Subject: [PATCH 3/3] =?UTF-8?q?fix(anchor):=20exclude=20and=20reject=20unp?= =?UTF-8?q?aired=20UTF-16=20surrogates=20in=20=C2=A71=20and=20serializers?= =?UTF-8?q?=20(#536)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Exclude U+D800 to U+DFFF from the BMP escape range in spec/registry-anchor-v1.md §1 (scoping BMP \uXXXX escapes to U+007F to U+D7FF and U+E000 to U+FFFF) and state that unpaired UTF-16 surrogates in strings or object keys are outside the anchor-leaf profile. - Reject unpaired UTF-16 surrogates in agentrust_trace.sign.anchor_bytes (raising UnanchorableValue) and trace_tests.inclusion.canonical_claim_bytes (raising InclusionError). - Add test_unpaired_surrogates_in_values_and_keys_are_rejected in tests/test_registry_anchor_canonical_rules.py. Signed-off-by: Raghuvar Durga <177662415+drk-ram@users.noreply.github.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- conformance/src/trace_tests/inclusion.py | 25 ++++++++++++++----- spec/registry-anchor-v1.md | 4 +-- src/agentrust_trace/sign.py | 23 ++++++++++++++++- tests/test_registry_anchor_canonical_rules.py | 23 ++++++++++++++--- 4 files changed, 63 insertions(+), 12 deletions(-) diff --git a/conformance/src/trace_tests/inclusion.py b/conformance/src/trace_tests/inclusion.py index ff6303e2..ded93563 100644 --- a/conformance/src/trace_tests/inclusion.py +++ b/conformance/src/trace_tests/inclusion.py @@ -37,16 +37,23 @@ class InclusionError(ValueError): def _outside_profile(value: Any) -> str | None: """Name the first value registry-anchor-v1 section 1 excludes, or None. - Non-integer numbers (NaN and the infinities among them) and integers outside - -(2**53 - 1) to 2**53 - 1 are outside the anchor-leaf profile: two - implementations of the four serialization rules write different bytes for - them, so a proof over them only proves what Python's ``json.dumps`` wrote. - Iterative, because the claim is attacker-controlled and may be deep. + Non-integer numbers (NaN and the infinities among them), integers outside + -(2**53 - 1) to 2**53 - 1, and unpaired UTF-16 surrogates are outside the + anchor-leaf profile: two implementations of the five serialization rules + write different bytes or cannot represent them, so a proof over them only + proves what Python's ``json.dumps`` wrote. Iterative, because the claim is + attacker-controlled and may be deep. """ stack = [value] while stack: v = stack.pop() - if isinstance(v, bool) or v is None or isinstance(v, str): + if isinstance(v, bool) or v is None: + continue + if isinstance(v, str): + try: + v.encode("utf-8") + except UnicodeEncodeError: + return "unpaired UTF-16 surrogate" continue if isinstance(v, int): if not -_SAFE_INT <= v <= _SAFE_INT: @@ -54,6 +61,12 @@ def _outside_profile(value: Any) -> str | None: elif isinstance(v, float): return f"non-integer number {v!r}" elif isinstance(v, dict): + for k in v: + if isinstance(k, str): + try: + k.encode("utf-8") + except UnicodeEncodeError: + return "unpaired UTF-16 surrogate in object key" stack.extend(v.values()) elif isinstance(v, (list, tuple)): stack.extend(v) diff --git a/spec/registry-anchor-v1.md b/spec/registry-anchor-v1.md index 2d5a1686..24d955e6 100644 --- a/spec/registry-anchor-v1.md +++ b/spec/registry-anchor-v1.md @@ -47,7 +47,7 @@ The unit of anchoring is the **complete signed claim object, signature included* - Object keys sorted lexicographically by Unicode code point, recursively (UTF-16 surrogate pairs are not used for key comparison; see §0). - Separators `","` and `":"`, no whitespace. -- String escaping: `"` (`U+0022`), `\` (`U+005C`), and the five control characters `U+0008`, `U+0009`, `U+000A`, `U+000C`, and `U+000D` use the two-character escapes `\"`, `\\`, `\b`, `\t`, `\n`, `\f`, and `\r`; `/` (`U+002F`) is not escaped; all other control characters (`U+0000` to `U+001F`) and all BMP characters from `U+007F` to `U+FFFF` (including `U+007F`, which is escaped as `\u007f`) use lowercase four-hex-digit `\uXXXX` escapes; supplementary-plane characters (`U+10000` to `U+10FFFF`) use lowercase UTF-16 surrogate pairs (a high surrogate in `U+D800` to `U+DBFF` followed by a low surrogate in `U+DC00` to `U+DFFF`, e.g. `\ud8xx\udcxx`); output is encoded as ASCII bytes. +- String escaping: `"` (`U+0022`), `\` (`U+005C`), and the five control characters `U+0008`, `U+0009`, `U+000A`, `U+000C`, and `U+000D` use the two-character escapes `\"`, `\\`, `\b`, `\t`, `\n`, `\f`, and `\r`; `/` (`U+002F`) is not escaped; all other control characters (`U+0000` to `U+001F`) and all BMP characters in `U+007F` to `U+D7FF` and `U+E000` to `U+FFFF` (including `U+007F`, which is escaped as `\u007f`) use lowercase four-hex-digit `\uXXXX` escapes; supplementary-plane characters (`U+10000` to `U+10FFFF`) use lowercase UTF-16 surrogate pairs (a high surrogate in `U+D800` to `U+DBFF` followed by a low surrogate in `U+DC00` to `U+DFFF`, e.g. `\ud8xx\udcxx`); output is encoded as ASCII bytes. - Integer zero is written `0` (never `-0`). - No trailing newline. @@ -57,7 +57,7 @@ In Python this is exactly: json.dumps(claim, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii") ``` -Claims MUST be JSON objects. TRACE Trust Records contain only strings, integers, booleans, nulls, arrays, and objects. Claims containing non-integer numbers are outside this profile, because cross-language float serialization is not canonical, and claims containing an integer outside -9007199254740991 to 9007199254740991 are outside it for the same reason: an implementation of the five rules above in a language whose only number type is the IEEE 754 double writes one value for two distinct integers. TRACE v0.2 §3.2.2 already holds a Trust Record to that range, so a schema-valid record cannot reach this; a claim that is not a Trust Record can, and this is where it is excluded. +Claims MUST be JSON objects. TRACE Trust Records contain only strings, integers, booleans, nulls, arrays, and objects. Claims containing non-integer numbers are outside this profile, because cross-language float serialization is not canonical, and claims containing an integer outside -9007199254740991 to 9007199254740991 are outside it for the same reason: an implementation of the five rules above in a language whose only number type is the IEEE 754 double writes one value for two distinct integers. Claims containing an unpaired UTF-16 surrogate (`U+D800` to `U+DFFF`) in a string or object key are also outside this profile, because languages whose string type is UTF-8 cannot represent an unpaired surrogate. TRACE v0.2 §3.2.2 already holds a Trust Record to that range, so a schema-valid record cannot reach this; a claim that is not a Trust Record can, and this is where it is excluded. ## 2. Leaf hash diff --git a/src/agentrust_trace/sign.py b/src/agentrust_trace/sign.py index b11dc03e..43daa705 100644 --- a/src/agentrust_trace/sign.py +++ b/src/agentrust_trace/sign.py @@ -419,6 +419,17 @@ class UnanchorableValue(ValueError): def _reject_unanchorable(value: Any, path: str = "$") -> None: if isinstance(value, bool): return + if isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeEncodeError: + raise UnanchorableValue( + f"{path} contains an unpaired UTF-16 surrogate, which registry-anchor-v1 " + "section 1 puts outside the profile: languages whose string type is UTF-8 " + "cannot represent an unpaired surrogate, so the digest is not reproducible " + "by a peer." + ) from None + return if isinstance(value, float): raise UnanchorableValue( f"{path} is a non-integer number ({value!r}). registry-anchor-v1 section 1 " @@ -438,8 +449,18 @@ def _reject_unanchorable(value: Any, path: str = "$") -> None: ) if isinstance(value, dict): for key, item in value.items(): + if isinstance(key, str): + try: + key.encode("utf-8") + except UnicodeEncodeError: + raise UnanchorableValue( + f"{path}.{key!r} contains an unpaired UTF-16 surrogate in an " + "object key, which registry-anchor-v1 section 1 puts outside the " + "profile: languages whose string type is UTF-8 cannot represent " + "an unpaired surrogate." + ) from None _reject_unanchorable(item, f"{path}.{key}") - elif isinstance(value, list): + elif isinstance(value, (list, tuple)): for index, item in enumerate(value): _reject_unanchorable(item, f"{path}[{index}]") diff --git a/tests/test_registry_anchor_canonical_rules.py b/tests/test_registry_anchor_canonical_rules.py index cd12aa6f..32e67c35 100644 --- a/tests/test_registry_anchor_canonical_rules.py +++ b/tests/test_registry_anchor_canonical_rules.py @@ -7,7 +7,7 @@ import pytest -from agentrust_trace.sign import anchor_bytes +from agentrust_trace.sign import UnanchorableValue, anchor_bytes REPO_ROOT = Path(__file__).resolve().parents[1] _INCLUSION_SPEC = importlib.util.spec_from_file_location( @@ -16,6 +16,7 @@ assert _INCLUSION_SPEC is not None and _INCLUSION_SPEC.loader is not None _INCLUSION_MODULE = importlib.util.module_from_spec(_INCLUSION_SPEC) _INCLUSION_SPEC.loader.exec_module(_INCLUSION_MODULE) +InclusionError = _INCLUSION_MODULE.InclusionError canonical_claim_bytes = _INCLUSION_MODULE.canonical_claim_bytes SERIALIZERS = (anchor_bytes, canonical_claim_bytes) @@ -120,6 +121,21 @@ def test_rule_4_integer_zero_is_written_without_minus_sign(serialize) -> None: assert serialize({"n": -0}) == b'{"n":0}' +@pytest.mark.parametrize("bad_str", ["\ud800", "\udbff", "\udc00", "\udfff", "a\ud800b"]) +def test_unpaired_surrogates_in_values_and_keys_are_rejected(bad_str: str) -> None: + """Unpaired UTF-16 surrogates (U+D800..U+DFFF) are rejected as outside the profile.""" + for claim in ( + {"s": bad_str}, + {"nested": [bad_str]}, + {bad_str: 1}, + {"nested": {bad_str: 1}}, + ): + with pytest.raises(UnanchorableValue, match="surrogate"): + anchor_bytes(claim) + with pytest.raises(InclusionError, match="surrogate"): + canonical_claim_bytes(claim) + + def test_registry_anchor_spec_documents_all_five_canonical_rules() -> None: """spec/registry-anchor-v1.md §1 states all five language-independent rules.""" doc = (REPO_ROOT / "spec" / "registry-anchor-v1.md").read_text(encoding="utf-8") @@ -132,8 +148,8 @@ def test_registry_anchor_spec_documents_all_five_canonical_rules() -> None: "`/` (`U+002F`) is not escaped", "(`U+0000` to `U+001F`)", ( - "from `U+007F` to `U+FFFF` (including `U+007F`, which is escaped as " - "`\\u007f`) use lowercase four-hex-digit `\\uXXXX` escapes" + "in `U+007F` to `U+D7FF` and `U+E000` to `U+FFFF` (including `U+007F`, " + "which is escaped as `\\u007f`) use lowercase four-hex-digit `\\uXXXX` escapes" ), ( "(`U+10000` to `U+10FFFF`) use lowercase UTF-16 surrogate pairs " @@ -142,6 +158,7 @@ def test_registry_anchor_spec_documents_all_five_canonical_rules() -> None: ), "Integer zero is written `0` (never `-0`).", "No trailing newline.", + "unpaired UTF-16 surrogate (`U+D800` to `U+DFFF`)", ): assert required_fragment in section_1, ( f"spec/registry-anchor-v1.md §1 is missing {required_fragment!r}"