Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ agentmetry dashboard
| Antigravity | Tool calls and inferred approvals | `scripts/install_antigravity_hooks.ps1` on Windows. `hooks install` and `scripts/install.sh` do not write these. |
| MCP servers | Every stdio `tools/call`, plus a `tools/list` fingerprint | `agentmetry mcp-proxy -- <server command>` |
| CrewAI, OpenSRE | Framework adapters | [adapters/crewai](adapters/crewai/), [adapters/opensre](adapters/opensre/) |
| Microsoft Agent Governance Toolkit files | Import after the file's own chain verifies | `agentmetry import-agt` |
| Microsoft Agent Governance Toolkit files | Import policy and tool records with a per-file chain check and optional HMAC verification | `agentmetry import-agt` |

Any agent that acts on your machine can be recorded through the ingest API, not only coding agents. [Agent coverage](docs/agent-coverage.md) says what is supported, what is possible and what is impossible, for local agents such as OpenClaw and Hermes, frameworks such as LangGraph, and hosted agents such as the Grok app. The detections are written for agents that act on an endpoint.

Expand Down Expand Up @@ -158,7 +158,7 @@ These are the numbers from commands, not from a slide.

Run `agentmetry benchmark` to check these expectations. The 14 rule IDs in the corpus include 13 published rules and the experimental rule. The Sigma generator also covers `host-subagent-swarm-burst` and `off-hours-activity`.

2905 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The
2911 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The
[detection pass CI run](https://github.com/blitzcrieg1/agentmetry/actions/runs/44312544675)
reported 2885 passed and 19 skipped on Linux, 2898 passed and 6 skipped on Windows,
and 84% line coverage on Linux
Expand Down
14 changes: 5 additions & 9 deletions apps/orchestrator/agentmetry/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1033,9 +1033,7 @@ def cmd_export(args: argparse.Namespace) -> int:
def cmd_import_agt(args: argparse.Namespace) -> int:
"""Ingest a Microsoft Agent Governance Toolkit audit file into the trail.

AGT decides allow or deny per call; this says what a session of those calls
adds up to. In testing it read three calls AGT had individually allowed and
raised one critical `credential-exfil` across them.
Import policy evidence and reported tool activity under distinct action types.
"""
import os

Expand Down Expand Up @@ -1068,11 +1066,10 @@ def cmd_import_agt(args: argparse.Namespace) -> int:
return 1

print(f"OK — {result.message}")
if key is None:
if result.entries and not result.signatures_checked:
print(
" No HMAC key supplied, so signatures were not checked. Hashes and "
"chain linkage were, which catches editing and reordering. Pass --key "
"or set AGENTMETRY_AGT_HMAC_KEY to check forgery too."
" Source authenticity unverified. Only content hashes and chain linkage "
"were checked. Use AGENTMETRY_AGT_HMAC_KEY to verify HMAC with a trusted shared key."
)

if args.dry_run:
Expand All @@ -1090,8 +1087,7 @@ def cmd_import_agt(args: argparse.Namespace) -> int:
for event in events:
append_chained_line(trail, event)
print(f" Appended {len(events)} event(s) to {trail}")
print(" Marked source.tier=external, app=agt: Agentmetry read this record "
"rather than observing the calls.")
print(" Marked source.tier=external, app=agt: these are imported source records.")
return 0


Expand Down
108 changes: 56 additions & 52 deletions apps/orchestrator/agentmetry/core/audit/adapters/agt.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,22 +85,14 @@
_ACTION_TYPE = {
"tool_invocation": "tool_called",
"tool_blocked": "tool_denied",
"policy_evaluation": "tool_called",
"policy_evaluation": "policy_evaluation",
"approval_decision": "approval_decision",
"policy_violation": "tool_denied",
"rogue_detection": "detection",
"agent_invocation": "session_start",
}

#: AGT policy verdict -> Agentmetry action.outcome, when AGT's own `outcome`
#: is missing. AGT carries both a verdict (`action`) and a result (`outcome`);
#: Agentmetry has one field, and the result is the more factual of the two.
_OUTCOME = {
"allow": "success",
"deny": "denied",
"quarantine": "denied",
"warning": "success",
"audit": "success",
}
_DECISION_TYPES = {"policy_evaluation", "approval_decision"}


def content_hash(entry: dict[str, Any]) -> str:
Expand Down Expand Up @@ -148,15 +140,9 @@ def verify_agt_chain(
) -> AgtVerifyResult:
"""Check content hashes, chain linkage and (with a key) HMAC signatures.

Verified before ingest, never after. Importing someone else's audit record
into a hash-chained trail without checking it first would launder an
unverified claim into a chain that then vouches for it, and the chain would
be telling the truth about a lie.

The signature is optional because the key is optional. Without it the
content hashes and the linkage are still checkable, which catches editing
and reordering; only forgery by someone holding the key is out of reach,
and that is out of reach for AGT too.
Without a trusted key this checks internal consistency. Verification
covers the supplied file and cannot establish capture completeness or
continuity across rotated files.
"""
if not rows:
return AgtVerifyResult(ok=True, message="no AGT entries found", entries=0)
Expand Down Expand Up @@ -199,40 +185,21 @@ def verify_agt_chain(
signatures_checked=secret_key is not None,
)

checked = "hashes, chain and signatures" if secret_key is not None else "hashes and chain"
checked = "chain consistent and HMAC verified" if secret_key is not None else "chain consistent"
return AgtVerifyResult(
ok=True,
message=f"{len(rows)} AGT entries verified ({checked})",
message=f"{len(rows)} AGT entries checked ({checked})",
entries=len(rows),
signatures_checked=secret_key is not None,
)


def agt_to_canonical(
entry: dict[str, Any], *, host_id: str = "", fleet_id: str = ""
) -> dict[str, Any]:
"""Map one AGT entry onto an Agentmetry canonical event.

MITRE tagging and traits are computed here rather than carried across,
because AGT does not produce them and they are what lets the sequence rules
see this activity at all. That is the point of the adapter: AGT decides
allow or deny per call, and Agentmetry says what a session of those calls
adds up to.
"""
def _tool_evidence(resource: str, data: dict[str, Any]) -> dict[str, Any]:
from agentmetry.core.audit.detection.traits import classify_tool_call
from agentmetry.core.audit.atlas import attach_atlas
from agentmetry.core.audit.canonical import SCHEMA_VERSION
from agentmetry.core.audit.mitre import get_mitre_mapping

data = entry.get("data") if isinstance(entry.get("data"), dict) else {}
resource = str(entry.get("resource") or "")
command = str(data.get("command") or "")
event_type = str(entry.get("event_type") or "")
verdict = str(entry.get("action") or "")
outcome = str(entry.get("outcome") or "") or _OUTCOME.get(verdict, "success")

reason_parts = [p for p in (entry.get("policy_decision"), entry.get("matched_rule")) if p]

tool: dict[str, Any] = {
"name": resource.rsplit(".", 1)[-1] if resource else "",
"qualified": resource,
Expand All @@ -253,10 +220,41 @@ def agt_to_canonical(
if mapping:
tool["mitre"] = mapping
attach_atlas(tool)
return tool


def agt_to_canonical(
entry: dict[str, Any], *,
host_id: str = "", fleet_id: str = "", verification: AgtVerifyResult | None = None,
) -> dict[str, Any]:
"""Map one source record and carry the verification actually performed."""
from agentmetry.core.audit.canonical import SCHEMA_VERSION

data = entry.get("data") if isinstance(entry.get("data"), dict) else {}
resource = str(entry.get("resource") or "")
event_type = str(entry.get("event_type") or "")
action_type = _ACTION_TYPE.get(event_type, "agt_event")
source_action = str(entry.get("action") or "")
outcome = str(entry.get("outcome") or "")
verdict = source_action
reason = entry.get("policy_decision")
matched_rule = entry.get("matched_rule")
if action_type in _DECISION_TYPES:
# The wrapper records the requested action in "action". The approval
# response and resolved evaluation are separate source observations.
if action_type == "policy_evaluation":
outcome = str(entry.get("policy_decision") or outcome)
verdict = outcome
reason = data.get("reason")
matched_rule = matched_rule or data.get("rule")
default_outcome = "denied" if action_type == "tool_denied" else "unknown"
reason_parts = [str(p) for p in (reason, matched_rule) if p]

agent_did = str(entry.get("agent_did") or "")
chain_consistent = bool(verification and verification.ok and verification.entries)
signatures_checked = bool(chain_consistent and verification and verification.signatures_checked)

return {
event = {
"schema_version": SCHEMA_VERSION,
# AGT's entry_id is `audit_<16 hex>`, not a UUID, so it cannot be used
# as an event_id directly. Derived deterministically so re-importing the
Expand All @@ -267,30 +265,33 @@ def agt_to_canonical(
"timestamp_utc": str(entry.get("timestamp") or ""),
"host_id": host_id,
"fleet_id": fleet_id,
"source_topic": f"external/agt/{_ACTION_TYPE.get(event_type, event_type or 'tool_called')}",
"source_topic": f"external/agt/{action_type}",
"source": {"tier": "external", "app": "agt", "adapter": "agt_file"},
"actor": {"type": "agent", "id": agent_did, "role": "agent"},
"initiator": {"actor_type": "agent", "trigger": "agt", "operator_id": agent_did},
"action": {
"type": _ACTION_TYPE.get(event_type, "tool_called"),
"outcome": outcome,
"reason": "; ".join(str(p) for p in reason_parts),
"type": action_type,
"outcome": outcome or default_outcome,
"reason": "; ".join(reason_parts),
},
"agent": {"name": agent_did or "agt", "skill_id": ""},
"model": {"id": "", "provider": ""},
"tool": tool,
# Custody. Agentmetry did not observe this call; it read AGT's record of
# it. A trail that cannot say so is asserting more than it knows.
"provenance": {
"captured_by": "agent-governance-toolkit",
"agt_entry_id": entry.get("entry_id"),
"agt_content_hash": entry.get("content_hash"),
"agt_previous_hash": entry.get("previous_hash"),
"agt_event_type": event_type,
"agt_action": source_action,
"agt_verdict": verdict,
"verified_on_import": True,
"chain_consistent": chain_consistent,
"signatures_checked": signatures_checked,
"verified_on_import": signatures_checked,
},
}
if action_type not in _DECISION_TYPES and action_type != "agt_event":
event["tool"] = _tool_evidence(resource, data)
return event


def agt_file_to_canonical(
Expand All @@ -305,5 +306,8 @@ def agt_file_to_canonical(
result = verify_agt_chain(rows, secret_key)
if not result.ok:
return result, []
events = [agt_to_canonical(r, host_id=host_id, fleet_id=fleet_id) for r in rows]
events = [
agt_to_canonical(row, host_id=host_id, fleet_id=fleet_id, verification=result)
for row in rows
]
return result, events
2 changes: 2 additions & 0 deletions apps/orchestrator/tests/fixtures/agt_govern_approval.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"action": "write", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "674c109ce17e0de46569c7cfa88220b3be12a13b8cb19cec8b58a045eb921928", "data": {"approver": "system:auto-reject", "reason": "Synthetic approval rejected", "rule": "fixture-rule"}, "entry_id": "audit_5b575c16036a46e1", "environment": null, "event_type": "approval_decision", "matched_rule": null, "outcome": "rejected", "policy_decision": null, "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "99896d857e1066633d59d608f8da012cc3952df3ff7bda07c63b946322dcbd6b", "target_did": null, "timestamp": "2026-11-19T08:41:19.685673+00:00", "trace_id": null}
{"action": "write", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "7f578878cf64faa82a82a7e4d3e5729a0874ee786dd9de7583ff80ecf94e71e5", "data": {"evaluation_ms": 0.05, "reason": "Approval rejected by system:auto-reject: Synthetic approval rejected", "rule": "fixture-rule"}, "entry_id": "audit_a476f5e8ec37439b", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "deny", "policy_decision": "deny", "previous_hash": "674c109ce17e0de46569c7cfa88220b3be12a13b8cb19cec8b58a045eb921928", "resource": null, "sandbox_id": null, "session_id": null, "signature": "4cc24afab28ae8727799f76fb5ee93471d39ba38cae799b2d53155fe24c09fcf", "target_did": null, "timestamp": "2026-11-19T08:41:19.685935+00:00", "trace_id": null}
2 changes: 2 additions & 0 deletions apps/orchestrator/tests/fixtures/agt_govern_decisions.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"action": "read", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "42effbc295d0af7fff654ede97fd14a850bfb9bb9adb4b19158ed9e0243621a8", "data": {"evaluation_ms": 0.196, "reason": "No matching rules, using default", "rule": ""}, "entry_id": "audit_7b9b7cd7877c46b5", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "allow", "policy_decision": "allow", "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "75c09c6dd87a11d324953a05fa1065bbf7e3a71b4a8c1e0f94925620f2f0400a", "target_did": null, "timestamp": "2026-11-19T08:41:19.683487+00:00", "trace_id": null}
{"action": "write", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "cc381836dd80b3d1dd7100cd4409671c941c447ca6dae9b41d3f7615b3d6b9eb", "data": {"evaluation_ms": 0.066, "reason": "Rule fixture-rule matched", "rule": "fixture-rule"}, "entry_id": "audit_7b90c99013044c6d", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "deny", "policy_decision": "deny", "previous_hash": "42effbc295d0af7fff654ede97fd14a850bfb9bb9adb4b19158ed9e0243621a8", "resource": null, "sandbox_id": null, "session_id": null, "signature": "04e5ce9d6a287dfe8ce96a1e99dcc76afb703ff12757fb72beb7421e3c345d81", "target_did": null, "timestamp": "2026-11-19T08:41:19.683941+00:00", "trace_id": null}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"action": "read", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "9fd5cf6ac3978cdd1510af18b650a287f7d36b8cd303d0cd0fa3eaf1180ea2ba", "data": {}, "entry_id": "audit_0a12d152628e458c", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "allow", "policy_decision": "allow", "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "abce37a816840282c8ae904d38e47cf55d300e3bf9df113e1f0c739626d13d51", "target_did": null, "timestamp": "2026-11-19T08:41:19.686593+00:00", "trace_id": null}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"action": "read", "agent_did": "fixture-agent", "compute_driver": null, "content_hash": "b0087eea54bfb8494c4e8a2a1e7db1d195731323edaa81b420304ce25bdc0039", "data": {}, "entry_id": "audit_602b6570c35b4621", "environment": null, "event_type": "policy_evaluation", "matched_rule": null, "outcome": "deny", "policy_decision": "deny", "previous_hash": "", "resource": null, "sandbox_id": null, "session_id": null, "signature": "e9cba25fd67e83ed5552320bb69bd5998299a764c67fdb12b55c79da715041ef", "target_did": null, "timestamp": "2026-11-19T08:41:19.686810+00:00", "trace_id": null}
68 changes: 68 additions & 0 deletions apps/orchestrator/tests/fixtures/generate_agt_govern.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
"""Capture FileAuditSink fixtures using AGT commit f85abe1.

Run with a separate AGT installation and an empty output directory.
The fixed HMAC key is for these synthetic fixtures only.
"""

import json
import sys
from pathlib import Path

from agentmesh.governance.approval import AutoRejectApproval
from agentmesh.governance.audit import AuditLog
from agentmesh.governance.audit_backends import FileAuditSink
from agentmesh.governance.govern import govern

KEY = b"agentmetry-synthetic-agt-fixture-key"
POLICY = """
apiVersion: governance.toolkit/v1
name: fixture-policy
default_action: allow
rules:
- name: fixture-rule
condition: "action.type == 'write'"
action: %s
"""


def capture(output: Path) -> None:
invoked = []

def harmless_tool(action: str) -> str:
invoked.append(action)
return "synthetic result"

for name, action in (("decisions", "deny"), ("approval", "require_approval")):
path = output / f"agt_govern_{name}.jsonl"
wrapped = govern(
harmless_tool,
policy=POLICY % action,
agent_id="fixture-agent",
audit_file=str(path),
audit_secret_key=KEY,
approval_handler=AutoRejectApproval("Synthetic approval rejected"),
on_deny=lambda decision: decision.action,
)
if action == "deny":
assert wrapped(action="read") == "synthetic result"
assert wrapped(action="write") == "deny"
assert FileAuditSink(path, secret_key=KEY).verify_integrity()[0]
assert invoked == ["read"]

live = output / "agt_govern_rotation_second.jsonl"
sink = FileAuditSink(live, secret_key=KEY, max_file_size=1)
audit = AuditLog(sink=sink)
for verdict in ("allow", "deny"):
audit.log("policy_evaluation", "fixture-agent", "read", outcome=verdict, policy_decision=verdict)
first, = output.glob("agt_govern_rotation_second.*.jsonl")
first.rename(output / "agt_govern_rotation_first.jsonl")
for path in output.glob("agt_govern_rotation_*.jsonl"):
assert FileAuditSink(path, secret_key=KEY).verify_integrity()[0]
assert json.loads(path.read_text(encoding="utf-8"))["previous_hash"] == ""


if __name__ == "__main__":
directory = Path(sys.argv[1]).resolve()
if not directory.is_dir() or any(directory.iterdir()):
raise SystemExit("Use an empty output directory for a fresh fixture capture.")
capture(directory)
Loading
Loading