diff --git a/README.md b/README.md index 3d51eda..c35c59d 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ These are the numbers from commands, not from a slide. Run `agentmetry benchmark` to check these expectations. The 14 rule IDs in the corpus include 13 published rules and the experimental rule. The Sigma generator also covers `host-subagent-swarm-burst` and `off-hours-activity`. -2911 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The +2917 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The [detection pass CI run](https://github.com/blitzcrieg1/agentmetry/actions/runs/44312544675) reported 2885 passed and 19 skipped on Linux, 2898 passed and 6 skipped on Windows, and 84% line coverage on Linux diff --git a/apps/orchestrator/agentmetry/api/routes/audit.py b/apps/orchestrator/agentmetry/api/routes/audit.py index dc951ab..ec483ca 100644 --- a/apps/orchestrator/agentmetry/api/routes/audit.py +++ b/apps/orchestrator/agentmetry/api/routes/audit.py @@ -439,9 +439,9 @@ async def audit_session_tree( ): """One session folded into investigation-tree nodes. - Each ask, its answer and the call it led to become one node, so a detection - that cites the ask and a policy verdict recorded on it land on the call - they belong to. A session longer than `limit` is windowed on `around_utc` + Claude request and result nodes carry exact invocation links where + available. Legacy asks can be folded with their answers and calls. + A session longer than `limit` is windowed on `around_utc` (a detection's first_seen_utc) when given. See `core/audit/session_tree.py`. """ from agentmetry.core.audit.session_tree import session_tree diff --git a/apps/orchestrator/agentmetry/core/audit/session_tree.py b/apps/orchestrator/agentmetry/core/audit/session_tree.py index fce0912..c01161c 100644 --- a/apps/orchestrator/agentmetry/core/audit/session_tree.py +++ b/apps/orchestrator/agentmetry/core/audit/session_tree.py @@ -1,18 +1,13 @@ """One session as the nodes of an investigation tree. -A hook records a tool call as up to three events: the agent asking -(`approval_request`), the answer (`approval_response`) and the call itself -(`tool_called`). A detection may cite any of them, and the DLP and tool policy -verdicts sit on the ask, not the call. Drawn one event per node, a session -shows every call two or three times and splits each verdict from the call it -judged. So the three are folded into one node here, joined on the input hash -the hook computes for all of them, and the node keeps every event id it -absorbed so a cited id still finds it. - -Cursor asks as `cursor.Shell` and runs as `shell.run`; the hash is the same, -which is why the join is on the hash and not the tool name. An ask with no call -after it stays a node of its own: that is a denial, or a call the agent -abandoned, and either is worth seeing. +Claude requests and reported results retain their own nodes and event IDs. +Only an unambiguous native invocation links those observations. Permission +responses remain separate when the source supplies no invocation ID for them. + +Legacy adapters can fold an ask, answer and call into one node with all their +event IDs when there is one pending ask for the hash in the source session. +Cursor asks as `cursor.Shell` and runs as `shell.run`, with the same input hash. +An ask with no call stays visible on its own. Nothing here reads command text the trail does not already hold. `command` is present only when the operator set `AGENTMETRY_LOG_COMMANDS`. @@ -22,6 +17,8 @@ from typing import Any +from agentmetry.core.audit.invocation import normalize_claude_tool_metadata + #: Long enough for a real shell line, short enough that one pasted script does #: not make the payload the size of the session. COMMAND_LIMIT = 4000 @@ -112,38 +109,94 @@ def _answer(event: dict[str, Any]) -> str: return "inferred" if inferred else "approved" +def _invocation_key(event: dict[str, Any], tool: dict[str, Any]) -> tuple[str, ...] | None: + # The tree reads one recorder's local index. The host and correlation ID + # constrain this comparison further and do not establish device identity. + source = event["source"] + key = ( + event.get("host_id"), event.get("correlation_id"), event.get("session_id"), + source.get("app"), source.get("adapter"), tool["qualified"], tool["server"], + tool["invocation"].get("id"), + ) + return key if all(isinstance(part, str) and part for part in key) else None + + +def _link_invocations(groups: dict[tuple[str, ...], list[dict[str, Any]]]) -> None: + for group in groups.values(): + if len(group) == 1: + continue + by_phase = {node["tool"]["invocation"]["phase"]: node["id"] for node in group} + request_id = by_phase.get("request") + result_id = by_phase.get("result") + if len(group) == 2 and request_id and result_id and request_id != result_id: + evidence = { + "status": "linked", + "request_event_id": request_id, + "result_event_id": result_id, + } + else: + evidence = {"status": "ambiguous"} + for node in group: + node["invocation_evidence"] = evidence.copy() + + def fold(events: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Fold a time-ordered session into tree nodes, asks joined to their calls.""" + """Project a session window and link the evidence available within it.""" nodes: list[dict[str, Any]] = [] - pending: dict[str, dict[str, Any]] = {} # input_hash -> the ask waiting for its call + invocations: dict[tuple[str, ...], list[dict[str, Any]]] = {} + pending: dict[tuple[str, ...], dict[str, Any] | None] = {} for event in events: action = (event.get("action") or {}).get("type", "") if action == "heartbeat": continue + kind = _kind(event) + node = _node(event, kind) + source = event.get("source") or {} + if source.get("app") == "claude": + if source.get("adapter") == "claude_hook" and action in ("approval_request", "tool_called"): + metadata = normalize_claude_tool_metadata(event.get("tool")) + phase = "request" if action == "approval_request" else "result" + if metadata and metadata["invocation"]["phase"] == phase: + node["tool"].update(metadata) + node["invocation_evidence"] = {"status": "unresolved"} + key = _invocation_key(event, node["tool"]) + if key is not None: + invocations.setdefault(key, []).append(node) + if action == "approval_response": + node["approval"] = _answer(event) + nodes.append(node) + continue + + input_hash = (event.get("gated_action") or {}).get("input_hash") if action == "approval_response" else ( + node["tool"]["input_hash"] + ) + scope = ( + str(event.get("host_id") or ""), str(event.get("correlation_id") or ""), node["app"], str(input_hash or ""), + ) if action == "approval_response": - gated = event.get("gated_action") or {} - ask = pending.get(str(gated.get("input_hash") or "")) + ask = pending.get(scope) if ask is None: - continue # an answer to an ask outside this window says nothing alone + if scope in pending: # two asks for the same hash leave the answer unlinked + node["approval"] = _answer(event) + nodes.append(node) + continue ask["approval"] = _answer(event) - ask["event_ids"].append(str(event.get("event_id") or "")) + ask["event_ids"].extend(node["event_ids"]) if ask["approval"] in ("denied", "expired"): if ask["approval"] == "denied": ask["kind"] = "denied" - pending.pop(ask["tool"]["input_hash"], None) + pending.pop(scope) continue - kind = _kind(event) if kind == "call": - call = _node(event, "call") - input_hash = call["tool"]["input_hash"] - ask = pending.pop(input_hash, None) if input_hash else None + ask = pending.get(scope) if ask is not None: + pending.pop(scope) ask["kind"] = "call" - ask["event_ids"].extend(call["event_ids"]) - ask["outcome"] = call["outcome"] - ask["reason"] = call["reason"] or ask["reason"] + ask["event_ids"].extend(node["event_ids"]) + ask["outcome"] = node["outcome"] + ask["reason"] = node["reason"] or ask["reason"] ask["asked_as"] = ask["tool"]["qualified"] - merged_tool = call["tool"] + merged_tool = node["tool"] # The ask may hold the command when the call does not, or the # other way round; keep whichever was recorded. if "command" not in merged_tool and "command" in ask["tool"]: @@ -152,18 +205,15 @@ def fold(events: list[dict[str, Any]]) -> list[dict[str, Any]]: merged_tool["mitre"] = ask["tool"]["mitre"] ask["tool"] = merged_tool for key in ("dlp", "tool_policy"): - if key in call and key not in ask: - ask[key] = call[key] - ask["id"] = call["id"] + if key in node and key not in ask: + ask[key] = node[key] + ask["id"] = node["id"] continue - nodes.append(call) - continue - node = _node(event, kind) nodes.append(node) - if kind == "ask" and node["tool"]["input_hash"]: - # A second ask with the same input before any call replaces the - # first as the one the next call answers; the first stays a node. - pending[node["tool"]["input_hash"]] = node + if kind == "ask" and input_hash: + # A collision stays ambiguous for the rest of this session window. + pending[scope] = None if scope in pending else node + _link_invocations(invocations) return nodes diff --git a/apps/orchestrator/tests/test_claude_invocation.py b/apps/orchestrator/tests/test_claude_invocation.py index 7fd1d7c..fcf9d32 100644 --- a/apps/orchestrator/tests/test_claude_invocation.py +++ b/apps/orchestrator/tests/test_claude_invocation.py @@ -107,6 +107,20 @@ def test_contract_request_result_survive_http_and_persistence(recorder): for event in events: if event["tool"]["invocation"]["phase"] == "result": assert event["tool"]["result"] == {"status": "reported_success"} + + tree = client.get("/api/v1/audit/session/synthetic-claude-session/tree") + assert tree.status_code == 200 + nodes = [node for node in tree.json()["nodes"] if "invocation" in node["tool"]] + assert len(nodes) == 4 + for identifier in ("toolu_synthetic_001", "toolu_synthetic_002"): + expected = { + event["tool"]["invocation"]["phase"] + "_event_id": event["event_id"] + for event in indexed if event["tool"]["invocation"]["id"] == identifier + } + matching = [node for node in nodes if node["tool"]["invocation"]["id"] == identifier] + assert len(matching) == 2 + assert all(node["invocation_evidence"] == {"status": "linked", **expected} for node in matching) + assert _MARKER not in tree.text assert _MARKER not in output.read_text() assert "printf synthetic" not in output.read_text() diff --git a/apps/orchestrator/tests/test_session_tree.py b/apps/orchestrator/tests/test_session_tree.py index 6e88da6..70fb605 100644 --- a/apps/orchestrator/tests/test_session_tree.py +++ b/apps/orchestrator/tests/test_session_tree.py @@ -127,6 +127,134 @@ def test_no_command_appears_unless_the_trail_recorded_one_and_long_ones_are_cut( assert nodes[1]["tool"]["command_truncated"] is True +def _claude(event_id, invocation_id, ts, *, hook="PreToolUse", **extra): + request = hook == "PreToolUse" + failed = hook == "PostToolUseFailure" + tool = _shell("Bash", "same-synthetic-hash") + tool["server"] = "claude" + tool["invocation"] = { + "id": invocation_id, "phase": "request" if request else "result", + "hook_event": hook, "provenance": "harness_reported", + } + if not request: + tool["result"] = { + "status": "reported_failure" if failed else "reported_success", "duration_ms": 0, + } + extra.setdefault("source", {"app": "claude", "adapter": "claude_hook"}) + extra.setdefault("session_id", "claude-session") + return _ev( + event_id, "approval_request" if request else "tool_called", ts, + "pending" if request else ("error" if failed else "success"), tool=tool, **extra, + ) + + +def test_invocation_evidence_distinguishes_concurrent_identical_arguments(): + policy = {"rule_id": "synthetic-policy", "action": "allow", "mode": "log", "blocked": False} + answer = _ev( + "answer", "approval_response", "00:02", reason="inferred:tool_ran_after_ask", + source={"app": "claude", "adapter": "claude_inferred"}, + gated_action={"input_hash": "same-synthetic-hash"}, + ) + events = [ + _claude("result-b", "call-b", "00:00", hook="PostToolUseFailure"), + _claude("request-a", "call-a", "00:01", tool_policy=policy), + answer, + _claude("request-b", "call-b", "00:03"), + _claude("result-a", "call-a", "00:04", hook="PostToolUse"), + ] + + nodes = {node["id"]: node for node in fold(events)} + + assert set(nodes) == {event["event_id"] for event in events} + for name in ("a", "b"): + request_id, result_id = f"request-{name}", f"result-{name}" + evidence = {"status": "linked", "request_event_id": request_id, "result_event_id": result_id} + assert nodes[request_id]["invocation_evidence"] == evidence + assert nodes[result_id]["invocation_evidence"] == evidence + assert nodes[request_id]["event_ids"] == [request_id] + assert nodes[result_id]["event_ids"] == [result_id] + assert nodes[result_id]["approval"] is None + assert nodes["request-a"]["tool_policy"] == policy + assert "tool_policy" not in nodes["result-a"] + assert nodes["answer"]["approval"] == "inferred" + assert "invocation_evidence" not in nodes["answer"] + assert nodes["result-a"]["tool"]["result"]["status"] == "reported_success" + assert nodes["result-b"]["tool"]["result"]["status"] == "reported_failure" + + +def test_invocation_evidence_leaves_missing_ids_and_results_unresolved(): + orphan = _claude("orphan", "orphan-id", "00:03", hook="PostToolUse") + nodes = fold([ + _claude("request", "call-a", "00:00"), + _claude("request-without-id", None, "00:01"), + _claude("result-without-id", None, "00:02", hook="PostToolUse"), + orphan, + ]) + + assert [node["invocation_evidence"] for node in nodes] == [{"status": "unresolved"}] * 4 + assert "result" not in nodes[0]["tool"] + assert "id" not in nodes[2]["tool"]["invocation"] + assert nodes[2]["tool"]["result"]["status"] == "reported_success" + assert nodes[3]["tool"]["invocation"]["id"] == "orphan-id" + + +def test_invocation_evidence_keeps_duplicate_or_conflicting_results_ambiguous(): + nodes = fold([ + _claude("request", "call-a", "00:00"), + _claude("result", "call-a", "00:01", hook="PostToolUse"), + _claude("duplicate", "call-a", "00:02", hook="PostToolUse"), + _claude("conflicting", "call-a", "00:03", hook="PostToolUseFailure"), + ]) + + assert [node["invocation_evidence"] for node in nodes] == [{"status": "ambiguous"}] * 4 + assert nodes[-1]["tool"]["result"]["status"] == "reported_failure" + assert "result" not in nodes[0]["tool"] + + +def test_invocation_evidence_requires_the_full_recorder_scope_and_tool(): + request = _claude("request", "call-a", "00:00") + result = _claude("result", "call-a", "00:01", hook="PostToolUse") + for fields in ( + {"session_id": "other-session"}, + {"session_id": ""}, + {"host_id": "OTHER"}, + {"correlation_id": "other-correlation"}, + {"source": {"app": "claude", "adapter": "other-adapter"}}, + {"source": {"app": "other-app", "adapter": "claude_hook"}}, + {"tool": {**result["tool"], "qualified": "Read"}}, + {"tool": {**result["tool"], "invocation": {**result["tool"]["invocation"], "id": "Call-a"}}}, + ): + request_node, result_node = fold([request, {**result, **fields}]) + assert request_node["invocation_evidence"] == {"status": "unresolved"} + assert result_node.get("invocation_evidence", {"status": "unresolved"}) == {"status": "unresolved"} + + +def test_invocation_evidence_projects_only_bounded_metadata(): + event = _claude("result", "call-a", "00:00", hook="PostToolUseFailure") + event["tool"]["invocation"]["arbitrary"] = "synthetic-private-marker" + event["tool"]["result"].update({"duration_ms": True, "error": "synthetic-private-marker"}) + node, = fold([event]) + + assert node["tool"]["result"] == {"status": "reported_failure", "error_kind": "tool_failure"} + assert "synthetic-private-marker" not in json.dumps(node) + + +def test_legacy_ambiguous_approval_does_not_attach_to_a_repeated_call(): + tool = _shell("shell.run", "same-hash") + nodes = fold([ + _ev("ask-1", "approval_request", "00:00", "pending", tool=tool), + _ev("other-host", "tool_called", "00:01", tool=tool, host_id="OTHER"), + _ev("ask-2", "approval_request", "00:02", "pending", tool=tool), + _ev("answer", "approval_response", "00:03", gated_action={"input_hash": "same-hash"}), + _ev("call", "tool_called", "00:04", tool=tool), + ]) + + assert [node["event_ids"] for node in nodes] == [ + ["ask-1"], ["other-host"], ["ask-2"], ["answer"], ["call"], + ] + assert [node["approval"] for node in nodes] == ["asked", None, "asked", "approved", None] + + @pytest.fixture def long_session(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): """2,100 heartbeats and 2,100 calls, then the call a detection cites.""" diff --git a/docs/claude-invocation-evidence.md b/docs/claude-invocation-evidence.md index 35e3da8..15d37f0 100644 --- a/docs/claude-invocation-evidence.md +++ b/docs/claude-invocation-evidence.md @@ -90,11 +90,21 @@ synthetic invocation, not a captured or executed session: ] ``` -One request and one result is the simple case. Multiple or conflicting -observations under the same key remain visible and ambiguous. The feature does -not infer attempts, suppress duplicates or provide a join engine. Arrival order -is not execution order; a request without a result remains unresolved. Never -substitute command similarity, argument hashes or a nearby timestamp for an ID. +The session-tree API retains the bounded `tool.invocation` and `tool.result` +blocks on their original event nodes. Within the returned session window, one +request and one result with the same exact key both get +`invocation_evidence.status: linked` and the two event IDs. The key uses the +local recorder index, event host, correlation ID, source session, source app +and adapter, tool identity and native invocation ID. Arrival order does not +affect the link. Multiple observations of the same phase stay visible with +`status: ambiguous`. A missing counterpart or key field gives +`status: unresolved`. Events outside the loaded window may be unavailable to +the comparison. + +Claude approval responses stay distinct from those links, with their +reported or inferred state. A request with no linked result remains +unresolved even when the trail contains an approval with a similar tool name, +argument hash or timestamp. ## Legacy approvals and remaining gaps