From d7f5626c8468168324de7f83277c54619f6d8037 Mon Sep 17 00:00:00 2001 From: "Ioannis L." <44038245+blitzcrieg1@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:09:06 +0200 Subject: [PATCH] Keep invocation evidence on session tree events Link Claude requests and reported results by the native invocation ID within their source and session. Keep unlinked approvals and ambiguous results visible, and guard against same-hash approval collisions. --- README.md | 2 +- .../agentmetry/api/routes/audit.py | 6 +- .../agentmetry/core/audit/session_tree.py | 126 +++++++++++------ .../tests/test_claude_invocation.py | 14 ++ apps/orchestrator/tests/test_session_tree.py | 128 ++++++++++++++++++ docs/claude-invocation-evidence.md | 20 ++- 6 files changed, 249 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index f5be9ca..38e1ff1 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ These are the numbers from commands, not from a slide. Run `agentmetry benchmark` to check these expectations. The 14 rule IDs in the corpus include 13 published rules and the experimental rule. The Sigma generator also covers `host-subagent-swarm-burst` and `off-hours-activity`. -2904 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The +Expected collection: 2910 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The [detection pass CI run](https://github.com/blitzcrieg1/agentmetry/actions/runs/44312544675) reported 2885 passed and 19 skipped on Linux, 2898 passed and 6 skipped on Windows, and 84% line coverage on Linux diff --git a/apps/orchestrator/agentmetry/api/routes/audit.py b/apps/orchestrator/agentmetry/api/routes/audit.py index dc951ab..ec483ca 100644 --- a/apps/orchestrator/agentmetry/api/routes/audit.py +++ b/apps/orchestrator/agentmetry/api/routes/audit.py @@ -439,9 +439,9 @@ async def audit_session_tree( ): """One session folded into investigation-tree nodes. - Each ask, its answer and the call it led to become one node, so a detection - that cites the ask and a policy verdict recorded on it land on the call - they belong to. A session longer than `limit` is windowed on `around_utc` + Claude request and result nodes carry exact invocation links where + available. Legacy asks can be folded with their answers and calls. + A session longer than `limit` is windowed on `around_utc` (a detection's first_seen_utc) when given. See `core/audit/session_tree.py`. """ from agentmetry.core.audit.session_tree import session_tree diff --git a/apps/orchestrator/agentmetry/core/audit/session_tree.py b/apps/orchestrator/agentmetry/core/audit/session_tree.py index fce0912..c01161c 100644 --- a/apps/orchestrator/agentmetry/core/audit/session_tree.py +++ b/apps/orchestrator/agentmetry/core/audit/session_tree.py @@ -1,18 +1,13 @@ """One session as the nodes of an investigation tree. -A hook records a tool call as up to three events: the agent asking -(`approval_request`), the answer (`approval_response`) and the call itself -(`tool_called`). A detection may cite any of them, and the DLP and tool policy -verdicts sit on the ask, not the call. Drawn one event per node, a session -shows every call two or three times and splits each verdict from the call it -judged. So the three are folded into one node here, joined on the input hash -the hook computes for all of them, and the node keeps every event id it -absorbed so a cited id still finds it. - -Cursor asks as `cursor.Shell` and runs as `shell.run`; the hash is the same, -which is why the join is on the hash and not the tool name. An ask with no call -after it stays a node of its own: that is a denial, or a call the agent -abandoned, and either is worth seeing. +Claude requests and reported results retain their own nodes and event IDs. +Only an unambiguous native invocation links those observations. Permission +responses remain separate when the source supplies no invocation ID for them. + +Legacy adapters can fold an ask, answer and call into one node with all their +event IDs when there is one pending ask for the hash in the source session. +Cursor asks as `cursor.Shell` and runs as `shell.run`, with the same input hash. +An ask with no call stays visible on its own. Nothing here reads command text the trail does not already hold. `command` is present only when the operator set `AGENTMETRY_LOG_COMMANDS`. @@ -22,6 +17,8 @@ from typing import Any +from agentmetry.core.audit.invocation import normalize_claude_tool_metadata + #: Long enough for a real shell line, short enough that one pasted script does #: not make the payload the size of the session. COMMAND_LIMIT = 4000 @@ -112,38 +109,94 @@ def _answer(event: dict[str, Any]) -> str: return "inferred" if inferred else "approved" +def _invocation_key(event: dict[str, Any], tool: dict[str, Any]) -> tuple[str, ...] | None: + # The tree reads one recorder's local index. The host and correlation ID + # constrain this comparison further and do not establish device identity. + source = event["source"] + key = ( + event.get("host_id"), event.get("correlation_id"), event.get("session_id"), + source.get("app"), source.get("adapter"), tool["qualified"], tool["server"], + tool["invocation"].get("id"), + ) + return key if all(isinstance(part, str) and part for part in key) else None + + +def _link_invocations(groups: dict[tuple[str, ...], list[dict[str, Any]]]) -> None: + for group in groups.values(): + if len(group) == 1: + continue + by_phase = {node["tool"]["invocation"]["phase"]: node["id"] for node in group} + request_id = by_phase.get("request") + result_id = by_phase.get("result") + if len(group) == 2 and request_id and result_id and request_id != result_id: + evidence = { + "status": "linked", + "request_event_id": request_id, + "result_event_id": result_id, + } + else: + evidence = {"status": "ambiguous"} + for node in group: + node["invocation_evidence"] = evidence.copy() + + def fold(events: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Fold a time-ordered session into tree nodes, asks joined to their calls.""" + """Project a session window and link the evidence available within it.""" nodes: list[dict[str, Any]] = [] - pending: dict[str, dict[str, Any]] = {} # input_hash -> the ask waiting for its call + invocations: dict[tuple[str, ...], list[dict[str, Any]]] = {} + pending: dict[tuple[str, ...], dict[str, Any] | None] = {} for event in events: action = (event.get("action") or {}).get("type", "") if action == "heartbeat": continue + kind = _kind(event) + node = _node(event, kind) + source = event.get("source") or {} + if source.get("app") == "claude": + if source.get("adapter") == "claude_hook" and action in ("approval_request", "tool_called"): + metadata = normalize_claude_tool_metadata(event.get("tool")) + phase = "request" if action == "approval_request" else "result" + if metadata and metadata["invocation"]["phase"] == phase: + node["tool"].update(metadata) + node["invocation_evidence"] = {"status": "unresolved"} + key = _invocation_key(event, node["tool"]) + if key is not None: + invocations.setdefault(key, []).append(node) + if action == "approval_response": + node["approval"] = _answer(event) + nodes.append(node) + continue + + input_hash = (event.get("gated_action") or {}).get("input_hash") if action == "approval_response" else ( + node["tool"]["input_hash"] + ) + scope = ( + str(event.get("host_id") or ""), str(event.get("correlation_id") or ""), node["app"], str(input_hash or ""), + ) if action == "approval_response": - gated = event.get("gated_action") or {} - ask = pending.get(str(gated.get("input_hash") or "")) + ask = pending.get(scope) if ask is None: - continue # an answer to an ask outside this window says nothing alone + if scope in pending: # two asks for the same hash leave the answer unlinked + node["approval"] = _answer(event) + nodes.append(node) + continue ask["approval"] = _answer(event) - ask["event_ids"].append(str(event.get("event_id") or "")) + ask["event_ids"].extend(node["event_ids"]) if ask["approval"] in ("denied", "expired"): if ask["approval"] == "denied": ask["kind"] = "denied" - pending.pop(ask["tool"]["input_hash"], None) + pending.pop(scope) continue - kind = _kind(event) if kind == "call": - call = _node(event, "call") - input_hash = call["tool"]["input_hash"] - ask = pending.pop(input_hash, None) if input_hash else None + ask = pending.get(scope) if ask is not None: + pending.pop(scope) ask["kind"] = "call" - ask["event_ids"].extend(call["event_ids"]) - ask["outcome"] = call["outcome"] - ask["reason"] = call["reason"] or ask["reason"] + ask["event_ids"].extend(node["event_ids"]) + ask["outcome"] = node["outcome"] + ask["reason"] = node["reason"] or ask["reason"] ask["asked_as"] = ask["tool"]["qualified"] - merged_tool = call["tool"] + merged_tool = node["tool"] # The ask may hold the command when the call does not, or the # other way round; keep whichever was recorded. if "command" not in merged_tool and "command" in ask["tool"]: @@ -152,18 +205,15 @@ def fold(events: list[dict[str, Any]]) -> list[dict[str, Any]]: merged_tool["mitre"] = ask["tool"]["mitre"] ask["tool"] = merged_tool for key in ("dlp", "tool_policy"): - if key in call and key not in ask: - ask[key] = call[key] - ask["id"] = call["id"] + if key in node and key not in ask: + ask[key] = node[key] + ask["id"] = node["id"] continue - nodes.append(call) - continue - node = _node(event, kind) nodes.append(node) - if kind == "ask" and node["tool"]["input_hash"]: - # A second ask with the same input before any call replaces the - # first as the one the next call answers; the first stays a node. - pending[node["tool"]["input_hash"]] = node + if kind == "ask" and input_hash: + # A collision stays ambiguous for the rest of this session window. + pending[scope] = None if scope in pending else node + _link_invocations(invocations) return nodes diff --git a/apps/orchestrator/tests/test_claude_invocation.py b/apps/orchestrator/tests/test_claude_invocation.py index 7fd1d7c..fcf9d32 100644 --- a/apps/orchestrator/tests/test_claude_invocation.py +++ b/apps/orchestrator/tests/test_claude_invocation.py @@ -107,6 +107,20 @@ def test_contract_request_result_survive_http_and_persistence(recorder): for event in events: if event["tool"]["invocation"]["phase"] == "result": assert event["tool"]["result"] == {"status": "reported_success"} + + tree = client.get("/api/v1/audit/session/synthetic-claude-session/tree") + assert tree.status_code == 200 + nodes = [node for node in tree.json()["nodes"] if "invocation" in node["tool"]] + assert len(nodes) == 4 + for identifier in ("toolu_synthetic_001", "toolu_synthetic_002"): + expected = { + event["tool"]["invocation"]["phase"] + "_event_id": event["event_id"] + for event in indexed if event["tool"]["invocation"]["id"] == identifier + } + matching = [node for node in nodes if node["tool"]["invocation"]["id"] == identifier] + assert len(matching) == 2 + assert all(node["invocation_evidence"] == {"status": "linked", **expected} for node in matching) + assert _MARKER not in tree.text assert _MARKER not in output.read_text() assert "printf synthetic" not in output.read_text() diff --git a/apps/orchestrator/tests/test_session_tree.py b/apps/orchestrator/tests/test_session_tree.py index 6e88da6..70fb605 100644 --- a/apps/orchestrator/tests/test_session_tree.py +++ b/apps/orchestrator/tests/test_session_tree.py @@ -127,6 +127,134 @@ def test_no_command_appears_unless_the_trail_recorded_one_and_long_ones_are_cut( assert nodes[1]["tool"]["command_truncated"] is True +def _claude(event_id, invocation_id, ts, *, hook="PreToolUse", **extra): + request = hook == "PreToolUse" + failed = hook == "PostToolUseFailure" + tool = _shell("Bash", "same-synthetic-hash") + tool["server"] = "claude" + tool["invocation"] = { + "id": invocation_id, "phase": "request" if request else "result", + "hook_event": hook, "provenance": "harness_reported", + } + if not request: + tool["result"] = { + "status": "reported_failure" if failed else "reported_success", "duration_ms": 0, + } + extra.setdefault("source", {"app": "claude", "adapter": "claude_hook"}) + extra.setdefault("session_id", "claude-session") + return _ev( + event_id, "approval_request" if request else "tool_called", ts, + "pending" if request else ("error" if failed else "success"), tool=tool, **extra, + ) + + +def test_invocation_evidence_distinguishes_concurrent_identical_arguments(): + policy = {"rule_id": "synthetic-policy", "action": "allow", "mode": "log", "blocked": False} + answer = _ev( + "answer", "approval_response", "00:02", reason="inferred:tool_ran_after_ask", + source={"app": "claude", "adapter": "claude_inferred"}, + gated_action={"input_hash": "same-synthetic-hash"}, + ) + events = [ + _claude("result-b", "call-b", "00:00", hook="PostToolUseFailure"), + _claude("request-a", "call-a", "00:01", tool_policy=policy), + answer, + _claude("request-b", "call-b", "00:03"), + _claude("result-a", "call-a", "00:04", hook="PostToolUse"), + ] + + nodes = {node["id"]: node for node in fold(events)} + + assert set(nodes) == {event["event_id"] for event in events} + for name in ("a", "b"): + request_id, result_id = f"request-{name}", f"result-{name}" + evidence = {"status": "linked", "request_event_id": request_id, "result_event_id": result_id} + assert nodes[request_id]["invocation_evidence"] == evidence + assert nodes[result_id]["invocation_evidence"] == evidence + assert nodes[request_id]["event_ids"] == [request_id] + assert nodes[result_id]["event_ids"] == [result_id] + assert nodes[result_id]["approval"] is None + assert nodes["request-a"]["tool_policy"] == policy + assert "tool_policy" not in nodes["result-a"] + assert nodes["answer"]["approval"] == "inferred" + assert "invocation_evidence" not in nodes["answer"] + assert nodes["result-a"]["tool"]["result"]["status"] == "reported_success" + assert nodes["result-b"]["tool"]["result"]["status"] == "reported_failure" + + +def test_invocation_evidence_leaves_missing_ids_and_results_unresolved(): + orphan = _claude("orphan", "orphan-id", "00:03", hook="PostToolUse") + nodes = fold([ + _claude("request", "call-a", "00:00"), + _claude("request-without-id", None, "00:01"), + _claude("result-without-id", None, "00:02", hook="PostToolUse"), + orphan, + ]) + + assert [node["invocation_evidence"] for node in nodes] == [{"status": "unresolved"}] * 4 + assert "result" not in nodes[0]["tool"] + assert "id" not in nodes[2]["tool"]["invocation"] + assert nodes[2]["tool"]["result"]["status"] == "reported_success" + assert nodes[3]["tool"]["invocation"]["id"] == "orphan-id" + + +def test_invocation_evidence_keeps_duplicate_or_conflicting_results_ambiguous(): + nodes = fold([ + _claude("request", "call-a", "00:00"), + _claude("result", "call-a", "00:01", hook="PostToolUse"), + _claude("duplicate", "call-a", "00:02", hook="PostToolUse"), + _claude("conflicting", "call-a", "00:03", hook="PostToolUseFailure"), + ]) + + assert [node["invocation_evidence"] for node in nodes] == [{"status": "ambiguous"}] * 4 + assert nodes[-1]["tool"]["result"]["status"] == "reported_failure" + assert "result" not in nodes[0]["tool"] + + +def test_invocation_evidence_requires_the_full_recorder_scope_and_tool(): + request = _claude("request", "call-a", "00:00") + result = _claude("result", "call-a", "00:01", hook="PostToolUse") + for fields in ( + {"session_id": "other-session"}, + {"session_id": ""}, + {"host_id": "OTHER"}, + {"correlation_id": "other-correlation"}, + {"source": {"app": "claude", "adapter": "other-adapter"}}, + {"source": {"app": "other-app", "adapter": "claude_hook"}}, + {"tool": {**result["tool"], "qualified": "Read"}}, + {"tool": {**result["tool"], "invocation": {**result["tool"]["invocation"], "id": "Call-a"}}}, + ): + request_node, result_node = fold([request, {**result, **fields}]) + assert request_node["invocation_evidence"] == {"status": "unresolved"} + assert result_node.get("invocation_evidence", {"status": "unresolved"}) == {"status": "unresolved"} + + +def test_invocation_evidence_projects_only_bounded_metadata(): + event = _claude("result", "call-a", "00:00", hook="PostToolUseFailure") + event["tool"]["invocation"]["arbitrary"] = "synthetic-private-marker" + event["tool"]["result"].update({"duration_ms": True, "error": "synthetic-private-marker"}) + node, = fold([event]) + + assert node["tool"]["result"] == {"status": "reported_failure", "error_kind": "tool_failure"} + assert "synthetic-private-marker" not in json.dumps(node) + + +def test_legacy_ambiguous_approval_does_not_attach_to_a_repeated_call(): + tool = _shell("shell.run", "same-hash") + nodes = fold([ + _ev("ask-1", "approval_request", "00:00", "pending", tool=tool), + _ev("other-host", "tool_called", "00:01", tool=tool, host_id="OTHER"), + _ev("ask-2", "approval_request", "00:02", "pending", tool=tool), + _ev("answer", "approval_response", "00:03", gated_action={"input_hash": "same-hash"}), + _ev("call", "tool_called", "00:04", tool=tool), + ]) + + assert [node["event_ids"] for node in nodes] == [ + ["ask-1"], ["other-host"], ["ask-2"], ["answer"], ["call"], + ] + assert [node["approval"] for node in nodes] == ["asked", None, "asked", "approved", None] + + @pytest.fixture def long_session(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """2,100 heartbeats and 2,100 calls, then the call a detection cites.""" diff --git a/docs/claude-invocation-evidence.md b/docs/claude-invocation-evidence.md index 35e3da8..15d37f0 100644 --- a/docs/claude-invocation-evidence.md +++ b/docs/claude-invocation-evidence.md @@ -90,11 +90,21 @@ synthetic invocation, not a captured or executed session: ] ``` -One request and one result is the simple case. Multiple or conflicting -observations under the same key remain visible and ambiguous. The feature does -not infer attempts, suppress duplicates or provide a join engine. Arrival order -is not execution order; a request without a result remains unresolved. Never -substitute command similarity, argument hashes or a nearby timestamp for an ID. +The session-tree API retains the bounded `tool.invocation` and `tool.result` +blocks on their original event nodes. Within the returned session window, one +request and one result with the same exact key both get +`invocation_evidence.status: linked` and the two event IDs. The key uses the +local recorder index, event host, correlation ID, source session, source app +and adapter, tool identity and native invocation ID. Arrival order does not +affect the link. Multiple observations of the same phase stay visible with +`status: ambiguous`. A missing counterpart or key field gives +`status: unresolved`. Events outside the loaded window may be unavailable to +the comparison. + +Claude approval responses stay distinct from those links, with their +reported or inferred state. A request with no linked result remains +unresolved even when the trail contains an approval with a similar tool name, +argument hash or timestamp. ## Legacy approvals and remaining gaps