From 748acec1678ce28169cd8c91934ce01a91789fd1 Mon Sep 17 00:00:00 2001 From: "Ioannis L." <44038245+blitzcrieg1@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:46:44 +0200 Subject: [PATCH] Keep AGT policy decisions separate from tool execution Preserve the source verdict and distinguish a consistent file chain from a keyed HMAC check. Cover decisions, approval, rotation and provenance with files captured from the upstream producer. --- README.md | 4 +- apps/orchestrator/agentmetry/cli/__init__.py | 14 +-- .../agentmetry/core/audit/adapters/agt.py | 108 ++++++++--------- .../tests/fixtures/agt_govern_approval.jsonl | 2 + .../tests/fixtures/agt_govern_decisions.jsonl | 2 + .../fixtures/agt_govern_rotation_first.jsonl | 1 + .../fixtures/agt_govern_rotation_second.jsonl | 1 + .../tests/fixtures/generate_agt_govern.py | 68 +++++++++++ apps/orchestrator/tests/test_agt_adapter.py | 110 ++++++++++++++++-- docs/agent-coverage.md | 2 +- docs/agentmetry-event-schema.md | 23 ++++ 11 files changed, 263 insertions(+), 72 deletions(-) create mode 100644 apps/orchestrator/tests/fixtures/agt_govern_approval.jsonl create mode 100644 apps/orchestrator/tests/fixtures/agt_govern_decisions.jsonl create mode 100644 apps/orchestrator/tests/fixtures/agt_govern_rotation_first.jsonl create mode 100644 apps/orchestrator/tests/fixtures/agt_govern_rotation_second.jsonl create mode 100644 apps/orchestrator/tests/fixtures/generate_agt_govern.py diff --git a/README.md b/README.md index f5be9ca3..3e14744f 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ agentmetry dashboard | Antigravity | Tool calls and inferred approvals | `scripts/install_antigravity_hooks.ps1` on Windows. `hooks install` and `scripts/install.sh` do not write these. | | MCP servers | Every stdio `tools/call`, plus a `tools/list` fingerprint | `agentmetry mcp-proxy -- ` | | CrewAI, OpenSRE | Framework adapters | [adapters/crewai](adapters/crewai/), [adapters/opensre](adapters/opensre/) | -| Microsoft Agent Governance Toolkit files | Import after the file's own chain verifies | `agentmetry import-agt` | +| Microsoft Agent Governance Toolkit files | Import policy and tool records with a per-file chain check and optional HMAC verification | `agentmetry import-agt` | Any agent that acts on your machine can be recorded through the ingest API, not only coding agents. [Agent coverage](docs/agent-coverage.md) says what is supported, what is possible and what is impossible, for local agents such as OpenClaw and Hermes, frameworks such as LangGraph, and hosted agents such as the Grok app. The detections are written for agents that act on an endpoint. @@ -158,7 +158,7 @@ These are the numbers from commands, not from a slide. Run `agentmetry benchmark` to check these expectations. The 14 rule IDs in the corpus include 13 published rules and the experimental rule. The Sigma generator also covers `host-subagent-swarm-burst` and `off-hours-activity`. -2904 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The +2910 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The [detection pass CI run](https://github.com/blitzcrieg1/agentmetry/actions/runs/44312544675) reported 2885 passed and 19 skipped on Linux, 2898 passed and 6 skipped on Windows, and 84% line coverage on Linux diff --git a/apps/orchestrator/agentmetry/cli/__init__.py b/apps/orchestrator/agentmetry/cli/__init__.py index 2b32895a..abeebe8e 100644 --- a/apps/orchestrator/agentmetry/cli/__init__.py +++ b/apps/orchestrator/agentmetry/cli/__init__.py @@ -1033,9 +1033,7 @@ def cmd_export(args: argparse.Namespace) -> int: def cmd_import_agt(args: argparse.Namespace) -> int: """Ingest a Microsoft Agent Governance Toolkit audit file into the trail. - AGT decides allow or deny per call; this says what a session of those calls - adds up to. In testing it read three calls AGT had individually allowed and - raised one critical `credential-exfil` across them. + Import policy evidence and reported tool activity under distinct action types. """ import os @@ -1068,11 +1066,10 @@ def cmd_import_agt(args: argparse.Namespace) -> int: return 1 print(f"OK — {result.message}") - if key is None: + if result.entries and not result.signatures_checked: print( - " No HMAC key supplied, so signatures were not checked. Hashes and " - "chain linkage were, which catches editing and reordering. Pass --key " - "or set AGENTMETRY_AGT_HMAC_KEY to check forgery too." + " Source authenticity unverified. Only content hashes and chain linkage " + "were checked. Use AGENTMETRY_AGT_HMAC_KEY to verify HMAC with a trusted shared key." ) if args.dry_run: @@ -1090,8 +1087,7 @@ def cmd_import_agt(args: argparse.Namespace) -> int: for event in events: append_chained_line(trail, event) print(f" Appended {len(events)} event(s) to {trail}") - print(" Marked source.tier=external, app=agt: Agentmetry read this record " - "rather than observing the calls.") + print(" Marked source.tier=external, app=agt: these are imported source records.") return 0 diff --git a/apps/orchestrator/agentmetry/core/audit/adapters/agt.py b/apps/orchestrator/agentmetry/core/audit/adapters/agt.py index b7688256..dc477601 100644 --- a/apps/orchestrator/agentmetry/core/audit/adapters/agt.py +++ b/apps/orchestrator/agentmetry/core/audit/adapters/agt.py @@ -85,22 +85,14 @@ _ACTION_TYPE = { "tool_invocation": "tool_called", "tool_blocked": "tool_denied", - "policy_evaluation": "tool_called", + "policy_evaluation": "policy_evaluation", + "approval_decision": "approval_decision", "policy_violation": "tool_denied", "rogue_detection": "detection", "agent_invocation": "session_start", } -#: AGT policy verdict -> Agentmetry action.outcome, when AGT's own `outcome` -#: is missing. AGT carries both a verdict (`action`) and a result (`outcome`); -#: Agentmetry has one field, and the result is the more factual of the two. -_OUTCOME = { - "allow": "success", - "deny": "denied", - "quarantine": "denied", - "warning": "success", - "audit": "success", -} +_DECISION_TYPES = {"policy_evaluation", "approval_decision"} def content_hash(entry: dict[str, Any]) -> str: @@ -148,15 +140,9 @@ def verify_agt_chain( ) -> AgtVerifyResult: """Check content hashes, chain linkage and (with a key) HMAC signatures. - Verified before ingest, never after. Importing someone else's audit record - into a hash-chained trail without checking it first would launder an - unverified claim into a chain that then vouches for it, and the chain would - be telling the truth about a lie. - - The signature is optional because the key is optional. Without it the - content hashes and the linkage are still checkable, which catches editing - and reordering; only forgery by someone holding the key is out of reach, - and that is out of reach for AGT too. + Without a trusted key this checks internal consistency. Verification + covers the supplied file and cannot establish capture completeness or + continuity across rotated files. """ if not rows: return AgtVerifyResult(ok=True, message="no AGT entries found", entries=0) @@ -199,40 +185,21 @@ def verify_agt_chain( signatures_checked=secret_key is not None, ) - checked = "hashes, chain and signatures" if secret_key is not None else "hashes and chain" + checked = "chain consistent and HMAC verified" if secret_key is not None else "chain consistent" return AgtVerifyResult( ok=True, - message=f"{len(rows)} AGT entries verified ({checked})", + message=f"{len(rows)} AGT entries checked ({checked})", entries=len(rows), signatures_checked=secret_key is not None, ) -def agt_to_canonical( - entry: dict[str, Any], *, host_id: str = "", fleet_id: str = "" -) -> dict[str, Any]: - """Map one AGT entry onto an Agentmetry canonical event. - - MITRE tagging and traits are computed here rather than carried across, - because AGT does not produce them and they are what lets the sequence rules - see this activity at all. That is the point of the adapter: AGT decides - allow or deny per call, and Agentmetry says what a session of those calls - adds up to. - """ +def _tool_evidence(resource: str, data: dict[str, Any]) -> dict[str, Any]: from agentmetry.core.audit.detection.traits import classify_tool_call from agentmetry.core.audit.atlas import attach_atlas - from agentmetry.core.audit.canonical import SCHEMA_VERSION from agentmetry.core.audit.mitre import get_mitre_mapping - data = entry.get("data") if isinstance(entry.get("data"), dict) else {} - resource = str(entry.get("resource") or "") command = str(data.get("command") or "") - event_type = str(entry.get("event_type") or "") - verdict = str(entry.get("action") or "") - outcome = str(entry.get("outcome") or "") or _OUTCOME.get(verdict, "success") - - reason_parts = [p for p in (entry.get("policy_decision"), entry.get("matched_rule")) if p] - tool: dict[str, Any] = { "name": resource.rsplit(".", 1)[-1] if resource else "", "qualified": resource, @@ -253,10 +220,41 @@ def agt_to_canonical( if mapping: tool["mitre"] = mapping attach_atlas(tool) + return tool + + +def agt_to_canonical( + entry: dict[str, Any], *, + host_id: str = "", fleet_id: str = "", verification: AgtVerifyResult | None = None, +) -> dict[str, Any]: + """Map one source record and carry the verification actually performed.""" + from agentmetry.core.audit.canonical import SCHEMA_VERSION + + data = entry.get("data") if isinstance(entry.get("data"), dict) else {} + resource = str(entry.get("resource") or "") + event_type = str(entry.get("event_type") or "") + action_type = _ACTION_TYPE.get(event_type, "agt_event") + source_action = str(entry.get("action") or "") + outcome = str(entry.get("outcome") or "") + verdict = source_action + reason = entry.get("policy_decision") + matched_rule = entry.get("matched_rule") + if action_type in _DECISION_TYPES: + # The wrapper records the requested action in "action". The approval + # response and resolved evaluation are separate source observations. + if action_type == "policy_evaluation": + outcome = str(entry.get("policy_decision") or outcome) + verdict = outcome + reason = data.get("reason") + matched_rule = matched_rule or data.get("rule") + default_outcome = "denied" if action_type == "tool_denied" else "unknown" + reason_parts = [str(p) for p in (reason, matched_rule) if p] agent_did = str(entry.get("agent_did") or "") + chain_consistent = bool(verification and verification.ok and verification.entries) + signatures_checked = bool(chain_consistent and verification and verification.signatures_checked) - return { + event = { "schema_version": SCHEMA_VERSION, # AGT's entry_id is `audit_<16 hex>`, not a UUID, so it cannot be used # as an event_id directly. Derived deterministically so re-importing the @@ -267,30 +265,33 @@ def agt_to_canonical( "timestamp_utc": str(entry.get("timestamp") or ""), "host_id": host_id, "fleet_id": fleet_id, - "source_topic": f"external/agt/{_ACTION_TYPE.get(event_type, event_type or 'tool_called')}", + "source_topic": f"external/agt/{action_type}", "source": {"tier": "external", "app": "agt", "adapter": "agt_file"}, "actor": {"type": "agent", "id": agent_did, "role": "agent"}, "initiator": {"actor_type": "agent", "trigger": "agt", "operator_id": agent_did}, "action": { - "type": _ACTION_TYPE.get(event_type, "tool_called"), - "outcome": outcome, - "reason": "; ".join(str(p) for p in reason_parts), + "type": action_type, + "outcome": outcome or default_outcome, + "reason": "; ".join(reason_parts), }, "agent": {"name": agent_did or "agt", "skill_id": ""}, "model": {"id": "", "provider": ""}, - "tool": tool, - # Custody. Agentmetry did not observe this call; it read AGT's record of - # it. A trail that cannot say so is asserting more than it knows. "provenance": { "captured_by": "agent-governance-toolkit", "agt_entry_id": entry.get("entry_id"), "agt_content_hash": entry.get("content_hash"), "agt_previous_hash": entry.get("previous_hash"), "agt_event_type": event_type, + "agt_action": source_action, "agt_verdict": verdict, - "verified_on_import": True, + "chain_consistent": chain_consistent, + "signatures_checked": signatures_checked, + "verified_on_import": signatures_checked, }, } + if action_type not in _DECISION_TYPES and action_type != "agt_event": + event["tool"] = _tool_evidence(resource, data) + return event def agt_file_to_canonical( @@ -305,5 +306,8 @@ def agt_file_to_canonical( result = verify_agt_chain(rows, secret_key) if not result.ok: return result, [] - events = [agt_to_canonical(r, host_id=host_id, fleet_id=fleet_id) for r in rows] + events = [ + agt_to_canonical(row, host_id=host_id, fleet_id=fleet_id, verification=result) + for row in rows + ] return result, events diff --git a/apps/orchestrator/tests/fixtures/agt_govern_approval.jsonl b/apps/orchestrator/tests/fixtures/agt_govern_approval.jsonl new file mode 100644 index 00000000..dc19b096 --- /dev/null +++ b/apps/orchestrator/tests/fixtures/agt_govern_approval.jsonl @@ -0,0 +1,2 @@ +{"action": "write", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "674c109ce17e0de46569c7cfa88220b3be12a13b8cb19cec8b58a045eb921928", "data": {"approver": "system:auto-reject", "reason": "Synthetic approval rejected", "rule": "fixture-rule"}, "entry_id": "audit_5b575c16036a46e1", "environment": null, "event_type": "approval_decision", "matched_rule": null, "outcome": "rejected", "policy_decision": null, "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "99896d857e1066633d59d608f8da012cc3952df3ff7bda07c63b946322dcbd6b", "target_did": null, "timestamp": "2026-11-19T08:41:19.685673+00:00", "trace_id": null} +{"action": "write", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "7f578878cf64faa82a82a7e4d3e5729a0874ee786dd9de7583ff80ecf94e71e5", "data": {"evaluation_ms": 0.05, "reason": "Approval rejected by system:auto-reject: Synthetic approval rejected", "rule": "fixture-rule"}, "entry_id": "audit_a476f5e8ec37439b", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "deny", "policy_decision": "deny", "previous_hash": "674c109ce17e0de46569c7cfa88220b3be12a13b8cb19cec8b58a045eb921928", "resource": null, "sandbox_id": null, "session_id": null, "signature": "4cc24afab28ae8727799f76fb5ee93471d39ba38cae799b2d53155fe24c09fcf", "target_did": null, "timestamp": "2026-11-19T08:41:19.685935+00:00", "trace_id": null} diff --git a/apps/orchestrator/tests/fixtures/agt_govern_decisions.jsonl b/apps/orchestrator/tests/fixtures/agt_govern_decisions.jsonl new file mode 100644 index 00000000..53cf4683 --- /dev/null +++ b/apps/orchestrator/tests/fixtures/agt_govern_decisions.jsonl @@ -0,0 +1,2 @@ +{"action": "read", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "42effbc295d0af7fff654ede97fd14a850bfb9bb9adb4b19158ed9e0243621a8", "data": {"evaluation_ms": 0.196, "reason": "No matching rules, using default", "rule": ""}, "entry_id": "audit_7b9b7cd7877c46b5", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "allow", "policy_decision": "allow", "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "75c09c6dd87a11d324953a05fa1065bbf7e3a71b4a8c1e0f94925620f2f0400a", "target_did": null, "timestamp": "2026-11-19T08:41:19.683487+00:00", "trace_id": null} +{"action": "write", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "cc381836dd80b3d1dd7100cd4409671c941c447ca6dae9b41d3f7615b3d6b9eb", "data": {"evaluation_ms": 0.066, "reason": "Rule fixture-rule matched", "rule": "fixture-rule"}, "entry_id": "audit_7b90c99013044c6d", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "deny", "policy_decision": "deny", "previous_hash": "42effbc295d0af7fff654ede97fd14a850bfb9bb9adb4b19158ed9e0243621a8", "resource": null, "sandbox_id": null, "session_id": null, "signature": "04e5ce9d6a287dfe8ce96a1e99dcc76afb703ff12757fb72beb7421e3c345d81", "target_did": null, "timestamp": "2026-11-19T08:41:19.683941+00:00", "trace_id": null} diff --git a/apps/orchestrator/tests/fixtures/agt_govern_rotation_first.jsonl b/apps/orchestrator/tests/fixtures/agt_govern_rotation_first.jsonl new file mode 100644 index 00000000..74b4cf97 --- /dev/null +++ b/apps/orchestrator/tests/fixtures/agt_govern_rotation_first.jsonl @@ -0,0 +1 @@ +{"action": "read", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "9fd5cf6ac3978cdd1510af18b650a287f7d36b8cd303d0cd0fa3eaf1180ea2ba", "data": {}, "entry_id": "audit_0a12d152628e458c", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "allow", "policy_decision": "allow", "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "abce37a816840282c8ae904d38e47cf55d300e3bf9df113e1f0c739626d13d51", "target_did": null, "timestamp": "2026-11-19T08:41:19.686593+00:00", "trace_id": null} diff --git a/apps/orchestrator/tests/fixtures/agt_govern_rotation_second.jsonl b/apps/orchestrator/tests/fixtures/agt_govern_rotation_second.jsonl new file mode 100644 index 00000000..07294013 --- /dev/null +++ b/apps/orchestrator/tests/fixtures/agt_govern_rotation_second.jsonl @@ -0,0 +1 @@ +{"action": "read", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "b0087eea54bfb8494c4e8a2a1e7db1d195731323edaa81b420304ce25bdc0039", "data": {}, "entry_id": "audit_602b6570c35b4621", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "deny", "policy_decision": "deny", "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "e9cba25fd67e83ed5552320bb69bd5998299a764c67fdb12b55c79da715041ef", "target_did": null, "timestamp": "2026-11-19T08:41:19.686810+00:00", "trace_id": null} diff --git a/apps/orchestrator/tests/fixtures/generate_agt_govern.py b/apps/orchestrator/tests/fixtures/generate_agt_govern.py new file mode 100644 index 00000000..0692b930 --- /dev/null +++ b/apps/orchestrator/tests/fixtures/generate_agt_govern.py @@ -0,0 +1,68 @@ +"""Capture FileAuditSink fixtures using AGT commit f85abe1. + +Run with a separate AGT installation and an empty output directory. +The fixed HMAC key is for these synthetic fixtures only. +""" + +import json +import sys +from pathlib import Path + +from agentmesh.governance.approval import AutoRejectApproval +from agentmesh.governance.audit import AuditLog +from agentmesh.governance.audit_backends import FileAuditSink +from agentmesh.governance.govern import govern + +KEY = b"agentmetry-synthetic-agt-fixture-key" +POLICY = """ +apiVersion: governance.toolkit/v1 +name: fixture-policy +default_action: allow +rules: + - name: fixture-rule + condition: "action.type == 'write'" + action: %s +""" + + +def capture(output: Path) -> None: + invoked = [] + + def harmless_tool(action: str) -> str: + invoked.append(action) + return "synthetic result" + + for name, action in (("decisions", "deny"), ("approval", "require_approval")): + path = output / f"agt_govern_{name}.jsonl" + wrapped = govern( + harmless_tool, + policy=POLICY % action, + agent_id="fixture-agent", + audit_file=str(path), + audit_secret_key=KEY, + approval_handler=AutoRejectApproval("Synthetic approval rejected"), + on_deny=lambda decision: decision.action, + ) + if action == "deny": + assert wrapped(action="read") == "synthetic result" + assert wrapped(action="write") == "deny" + assert FileAuditSink(path, secret_key=KEY).verify_integrity()[0] + assert invoked == ["read"] + + live = output / "agt_govern_rotation_second.jsonl" + sink = FileAuditSink(live, secret_key=KEY, max_file_size=1) + audit = AuditLog(sink=sink) + for verdict in ("allow", "deny"): + audit.log("policy_evaluation", "fixture-agent", "read", outcome=verdict, policy_decision=verdict) + first, = output.glob("agt_govern_rotation_second.*.jsonl") + first.rename(output / "agt_govern_rotation_first.jsonl") + for path in output.glob("agt_govern_rotation_*.jsonl"): + assert FileAuditSink(path, secret_key=KEY).verify_integrity()[0] + assert json.loads(path.read_text(encoding="utf-8"))["previous_hash"] == "" + + +if __name__ == "__main__": + directory = Path(sys.argv[1]).resolve() + if not directory.is_dir() or any(directory.iterdir()): + raise SystemExit("Use an empty output directory for a fresh fixture capture.") + capture(directory) diff --git a/apps/orchestrator/tests/test_agt_adapter.py b/apps/orchestrator/tests/test_agt_adapter.py index dbcac782..cccb6fe0 100644 --- a/apps/orchestrator/tests/test_agt_adapter.py +++ b/apps/orchestrator/tests/test_agt_adapter.py @@ -1,8 +1,9 @@ """Ingest Microsoft Agent Governance Toolkit audit files. -The fixtures are not hand-written. They were produced by the real -`FileAuditSink` from `agent-governance-toolkit-core` 5.0.0 in a throwaway venv, -and `AGT.verify_integrity()` returned True on them before they were copied here. +The original fixtures were produced by the real `FileAuditSink` from +`agent-governance-toolkit-core` 5.0.0. The `agt_govern_*` fixtures come from +the upstream version reviewed in discussion #4560. See +`fixtures/generate_agt_govern.py` for their capture script. That matters: an adapter tested only against its own idea of the format is testing that the author was consistent, not that the format is right. @@ -34,6 +35,8 @@ EXFIL = FIXTURES / "agt_filesink_exfil.jsonl" MIXED = FIXTURES / "agt_filesink_mixed.jsonl" KEY = b"scratch-key-not-a-secret" +GOVERN_KEY = b"agentmetry-synthetic-agt-fixture-key" +GOVERN_DECISIONS = FIXTURES / "agt_govern_decisions.jsonl" # ---------------------------------------------------------------------- @@ -69,12 +72,15 @@ def test_verification_passes_on_a_real_file(): def test_hashes_and_chain_are_checkable_without_the_key(): - """The key is optional; editing and reordering are caught regardless. Only - forgery by a key holder is out of reach, and that is out of reach for AGT - too.""" - result = verify_agt_chain(read_agt_file(EXFIL), None) + result, events = agt_file_to_canonical(EXFIL) assert result.ok assert not result.signatures_checked + assert "chain consistent" in result.message + for event in events: + provenance = event["provenance"] + assert provenance["chain_consistent"] is True + assert provenance["signatures_checked"] is False + assert provenance["verified_on_import"] is False def test_an_edited_entry_is_caught(tmp_path): @@ -103,9 +109,10 @@ def test_a_removed_entry_is_caught(): def test_a_wrong_key_is_caught(): - result = verify_agt_chain(read_agt_file(EXFIL), b"not-the-key") + result, events = agt_file_to_canonical(EXFIL, secret_key=b"not-the-key") assert not result.ok assert result.signature_failures + assert events == [] def test_an_unverifiable_file_yields_no_events(tmp_path): @@ -177,6 +184,93 @@ def test_custody_is_recorded_rather_than_implied(): assert event["source"]["app"] == "agt" assert event["provenance"]["captured_by"] == "agent-governance-toolkit" assert event["provenance"]["agt_content_hash"] + assert event["provenance"]["chain_consistent"] is False + assert event["provenance"]["signatures_checked"] is False + assert event["provenance"]["verified_on_import"] is False + + +def test_govern_records_decisions_without_tool_evidence(): + from agentmetry.core.audit.detection.engine import run_detections + + result, events = agt_file_to_canonical(GOVERN_DECISIONS, secret_key=GOVERN_KEY) + assert result.ok and result.signatures_checked + assert "HMAC verified" in result.message + assert [event["action"]["type"] for event in events] == ["policy_evaluation"] * 2 + assert [event["action"]["outcome"] for event in events] == ["allow", "deny"] + assert "fixture-rule" in events[1]["action"]["reason"] + assert [event["provenance"]["agt_action"] for event in events] == ["read", "write"] + assert [event["provenance"]["agt_verdict"] for event in events] == ["allow", "deny"] + rows = read_agt_file(GOVERN_DECISIONS) + assert [event["provenance"]["agt_entry_id"] for event in events] == [ + row["entry_id"] for row in rows + ] + assert all("tool" not in event for event in events) + for event in events: + provenance = event["provenance"] + assert provenance["chain_consistent"] is True + assert provenance["signatures_checked"] is True + assert provenance["verified_on_import"] is True + assert run_detections(events) == [] + + +def test_approval_and_resolved_evaluation_are_distinct_records(): + path = FIXTURES / "agt_govern_approval.jsonl" + result, events = agt_file_to_canonical(path, secret_key=GOVERN_KEY) + assert result.ok + assert [event["action"]["type"] for event in events] == [ + "approval_decision", "policy_evaluation", + ] + assert [event["action"]["outcome"] for event in events] == ["rejected", "deny"] + assert "Synthetic approval rejected" in events[0]["action"]["reason"] + assert all("tool" not in event for event in events) + assert len({event["event_id"] for event in events}) == 2 + + +def test_rotated_files_are_independent_chains(): + files = [FIXTURES / f"agt_govern_rotation_{suffix}.jsonl" for suffix in ("first", "second")] + for path in files: + result, events = agt_file_to_canonical(path, secret_key=GOVERN_KEY) + assert result.ok and result.signatures_checked + (event,) = events + assert event["provenance"]["agt_previous_hash"] == "" + result = verify_agt_chain(read_agt_file(files[0]) + read_agt_file(files[1]), GOVERN_KEY) + assert not result.ok + assert result.chain_failures == [1] + assert result.signature_failures == [] + + +def test_legacy_allow_without_a_tool_result_stays_unknown(): + row = {**read_agt_file(EXFIL)[0], "outcome": ""} + event = agt_to_canonical(row) + assert event["action"]["type"] == "tool_called" + assert event["action"]["outcome"] == "unknown" + assert event["provenance"]["agt_verdict"] == "allow" + + +def test_unknown_source_kind_does_not_become_a_tool_call(): + row = {**read_agt_file(EXFIL)[0], "event_type": "tool_called"} + event = agt_to_canonical(row) + assert event["action"]["type"] == "agt_event" + assert event["provenance"]["agt_event_type"] == "tool_called" + assert "tool" not in event + + +def test_cli_dry_run_distinguishes_chain_check_from_hmac(monkeypatch, capsys): + from agentmetry.cli import main + + monkeypatch.delenv("AGENTMETRY_AGT_HMAC_KEY", raising=False) + args = ["import-agt", str(GOVERN_DECISIONS), "--dry-run"] + assert main(args) == 0 + output = capsys.readouterr().out + assert "chain consistent" in output + assert "Source authenticity unverified" in output + assert "2 event(s) would be appended" in output + + monkeypatch.setenv("AGENTMETRY_AGT_HMAC_KEY", GOVERN_KEY.decode()) + assert main(args) == 0 + output = capsys.readouterr().out + assert "chain consistent and HMAC verified" in output + assert "Source authenticity unverified" not in output # ---------------------------------------------------------------------- diff --git a/docs/agent-coverage.md b/docs/agent-coverage.md index 873b04c2..8d955eb9 100644 --- a/docs/agent-coverage.md +++ b/docs/agent-coverage.md @@ -145,7 +145,7 @@ The process runs where you deploy it, so capture is an adapter in that process t Notes: - **Blocking is not a promise.** "Yes" means the framework documents a way for code at that point to stop the tool. Agentmetry's framework adapters are recorders. A blocking adapter would be new work, and a deny that the framework then ignores would repeat the Codex failure. -- **Microsoft Agent Governance Toolkit files** are a separate, shipped path: `agentmetry import-agt` reads a `FileAuditSink` JSONL file after the file's own chain verifies, tested in `test_agt_adapter.py`. It imports a file some other runtime wrote. It does not make Semantic Kernel, Agent Framework, LangGraph or CrewAI supported on their own. +- **Microsoft Agent Governance Toolkit files.** `agentmetry import-agt` reads a `FileAuditSink` JSONL file after checking hashes and chain links. Source authenticity requires HMAC verification with a trusted shared key. Policy and approval records carry their own action types. This is file import coverage. [AGT event mapping](agentmetry-event-schema.md#agent-governance-toolkit-file-import) - **Where the framework runs matters.** A LangGraph graph on your server is this category. The same graph on a vendor's hosted platform is category G. ## F. Self-hosted agent and workflow platforms diff --git a/docs/agentmetry-event-schema.md b/docs/agentmetry-event-schema.md index 8e5042a4..7c1a31e7 100644 --- a/docs/agentmetry-event-schema.md +++ b/docs/agentmetry-event-schema.md @@ -60,6 +60,29 @@ AGENTMETRY_SPLUNK_HEC_TOKEN=... `correlation_id` groups events from one agent session (IDE conversation or MCP proxy session). `session_id` is the dashboard/WebSocket session when applicable. +### Agent Governance Toolkit file import + +`agentmetry import-agt` imports AGT `FileAuditSink` records as external evidence. +`policy_evaluation` records the resolved policy verdict from `policy_decision` +when present. `approval_decision` records the approval outcome separately. +Both retain the source entry ID and requested action in `provenance`. +Only the evaluation represents the resolved policy decision. When the source +provides no shared identifier, the importer preserves the two records without +joining them. Neither action type asserts tool execution. + +Explicit `tool_invocation` records retain their reported outcome. A missing +outcome stays `unknown` even when a legacy producer records an allow verdict. +Unrecognized AGT event kinds use `agt_event` and retain their original kind +in provenance. + +`provenance.chain_consistent` reports the file-level hash and linkage check. +`provenance.signatures_checked` is true on imported records if HMAC was +checked with the supplied key. The legacy `verified_on_import` flag now +requires that keyed verification. With a trusted shared key, HMAC can authenticate +the covered fields. Each rotated file has its own chain. These checks cannot +establish complete capture or show whether another rotated file is missing. +Earlier imported records retain their original metadata. + ### Claude invocation observations (optional, additive) New Claude hook events can carry `tool.invocation` with `phase`, `hook_event`,