diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff0242..e42732f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ separately (currently `1.2.0`) and changes additively. ## [Unreleased] +### Added + +- **Repository execution preflight.** `agentmetry scan --preflight PATH` + inventories supported startup hooks, project MCP definitions, VS Code + folder-open tasks, npm install scripts and devcontainer lifecycle commands. + It shows unresolved references and partial coverage without exposing command + text or running repository code. + ## [0.12.2] - 2026-11-16 A privacy release. Both fixes apply only with command logging or full-argument diff --git a/README.md b/README.md index cf63255..a3038c6 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ To check what the agents on this machine are about to depend on, before anything agentmetry scan # MCP servers your agents launch, and AGENTS.md / CLAUDE.md files left in Downloads or on the Desktop agentmetry scan ~/.claude/skills # skills, rules and CLAUDE.md files: text that instructs the model instead of describing a tool agentmetry scan npm:NAME # one package name, before you install it +agentmetry scan --preflight ./repo # startup hooks, project MCP and setup declarations ``` It reads files and nothing else: it never starts a server or fetches a package. It exits 1 on a finding, and "no findings" means these checks found nothing, never that something is safe. The checks and their limits are in [docs/scanner.md](docs/scanner.md). @@ -156,7 +157,7 @@ These are the numbers from commands, not from a slide. That block is what `agentmetry benchmark` printed from the PyPI 0.9.4 package and from this tree. "Rules covered 13" counts rule ids named in corpus expectations. Twelve of those are published rules. The thirteenth is the experimental rule. Two published rules have no corpus case: `host-subagent-swarm-burst` and `off-hours-activity`. They are named in `apps/orchestrator/tools/generate_sigma_pack.py` so the Sigma pack still has them. -2523 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The +Expected collection: 2534 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The [0.12.1 release PR's CI run](https://github.com/blitzcrieg1/agentmetry/actions/runs/43610018759) on Python 3.12 reported 2443 passed and 19 skipped on Linux and macOS, 2456 passed and 6 skipped on Windows, and 83% line coverage on Linux diff --git a/apps/orchestrator/agentmetry/cli/__init__.py b/apps/orchestrator/agentmetry/cli/__init__.py index 205509b..a05eebd 100644 --- a/apps/orchestrator/agentmetry/cli/__init__.py +++ b/apps/orchestrator/agentmetry/cli/__init__.py @@ -1205,23 +1205,31 @@ def cmd_mcp(args: argparse.Namespace) -> int: def cmd_scan(args: argparse.Namespace) -> int: - """Check what an agent is about to depend on: MCP servers, skills, packages. + """Check agent dependencies or inventory repository execution declarations. Exits 1 when anything is found and 2 when nothing could be read. Unlike `mcp` and `posture`, which a health check runs in the background, this is run on purpose before something is installed, so a finding should stop the - script that ran it. See core/scanner/scan.py. + script that ran it. Preflight reports partial coverage with exit 2. """ import json as _json from pathlib import Path as _Path - from agentmetry.core.scanner.scan import run + if args.preflight is not None: + if args.target is not None or args.project is not None or args.known is not None: + print("--preflight PATH cannot be combined with a target, --project or --known", file=sys.stderr) + return 2 + from agentmetry.core.scanner.preflight import inspect_repository - report = run( - args.target, - project=_Path(args.project).resolve() if args.project else _Path.cwd(), - known_file=args.known, - ) + report = inspect_repository(_Path(args.preflight)) + else: + from agentmetry.core.scanner.scan import run + + report = run( + args.target, + project=_Path(args.project).resolve() if args.project else _Path.cwd(), + known_file=args.known, + ) if args.json: print(_json.dumps(report.to_json(), indent=2)) else: @@ -1810,7 +1818,7 @@ def main(argv: list[str] | None = None) -> int: ) scan = sub.add_parser( "scan", - help="check MCP servers, skills or a package name before an agent depends on them", + help="check agent dependencies or inventory what a repository can launch", ) scan.add_argument( "target", nargs="?", default=None, @@ -1820,6 +1828,7 @@ def main(argv: list[str] | None = None) -> int: ) scan.add_argument("--project", default=None, help="project root whose MCP config to include (default: cwd)") scan.add_argument("--known", default=None, help="file of extra known package names, one per line (npm:/pypi: or bare)") + scan.add_argument("--preflight", metavar="PATH", help="inventory repository execution declarations") scan.add_argument("--json", action="store_true", help="print the report as JSON") otel = sub.add_parser( "otel", help="receive Claude Code's native OpenTelemetry stream (no hooks needed)" diff --git a/apps/orchestrator/agentmetry/core/scanner/preflight.py b/apps/orchestrator/agentmetry/core/scanner/preflight.py new file mode 100644 index 0000000..1a5b368 --- /dev/null +++ b/apps/orchestrator/agentmetry/core/scanner/preflight.py @@ -0,0 +1,405 @@ +"""Inspect declarations that can start execution from a repository.""" + +from __future__ import annotations + +import json +import os +import re +import shlex +import stat +from dataclasses import asdict, dataclass, field +from itertools import islice +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any, Iterator + +MAX_ENTRIES = 12_000 +MAX_DEPTH = 32 +MAX_MANIFESTS = 512 +MAX_FILE_BYTES = 1_000_000 +MAX_TOTAL_BYTES = 16_000_000 +MAX_SURFACES = 1_000 +MAX_NOTICES = 100 + +SKIP_DIRS = frozenset({".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", ".next"}) +MANIFEST_TYPES = { + ".claude/settings.json": "claude", + ".claude/settings.local.json": "claude", + ".mcp.json": "mcp", + ".cursor/mcp.json": "mcp", + ".vscode/mcp.json": "vscode_mcp", + ".vscode/tasks.json": "tasks", + ".devcontainer.json": "devcontainer", + ".devcontainer/devcontainer.json": "devcontainer", +} +NPM_INSTALL_EVENTS = ("preinstall", "install", "postinstall", "prepublish", "preprepare", "prepare", "postprepare") +CONTAINER_EVENTS = ("initializeCommand", "onCreateCommand", "updateContentCommand", + "postCreateCommand", "postStartCommand", "postAttachCommand") +INTERPRETERS = frozenset({"node", "bun", "python", "python3", "bash", "sh", "pwsh", "powershell"}) +SCRIPT_SUFFIXES = frozenset({".js", ".mjs", ".cjs", ".ts", ".py", ".sh", ".ps1", ".bat", ".cmd"}) +SCRIPT_PATH = re.compile(r"[\w .:/\\-]+", flags=re.ASCII) + + +@dataclass(frozen=True) +class Reference: + status: str + path: str | None = None + size_bytes: int | None = None + + +@dataclass(frozen=True) +class Surface: + manifest: str + location: str + trigger: str + mechanism: str + reference: Reference + + +@dataclass +class PreflightReport: + target: str + manifests_read: int = 0 + surfaces: list[Surface] = field(default_factory=list) + unobserved: list[dict[str, str]] = field(default_factory=list) + omitted_notices: int = 0 + + def note(self, path: Path, reason: str) -> None: + if len(self.unobserved) < MAX_NOTICES: + self.unobserved.append({"path": path.as_posix(), "reason": reason}) + else: + self.omitted_notices += 1 + + def exit_code(self) -> int: + return 2 if self.unobserved or self.omitted_notices else 0 + + def to_json(self) -> dict[str, Any]: + return {"mode": "repository_preflight", **asdict(self), + "coverage": "partial" if self.exit_code() else "supported_manifests_read"} + + def lines(self) -> list[str]: + status = "Partial inventory" if self.exit_code() else "Inventory" + out = [f"Agentmetry repository preflight: {json.dumps(self.target)}", + f" {status}: {len(self.surfaces)} declarations in {self.manifests_read} supported manifests"] + for surface in self.surfaces: + ref = surface.reference + suffix = f" {json.dumps(ref.path)} ({ref.size_bytes} bytes)" if ref.path is not None else "" + out.append(f" {surface.trigger} | {surface.mechanism} | " + f"{json.dumps(surface.manifest)}:{surface.location} | {ref.status}{suffix}") + for issue in self.unobserved: + out.append(f" Unobserved: {json.dumps(issue['path'])} | {issue['reason']}") + if self.omitted_notices: + out.append(f" {self.omitted_notices} additional notices omitted") + out.append(" Declarations describe possible execution. Script contents and runtime behavior are unobserved.") + return out + + +def _jsonc(text: str) -> str: + """Remove comments and trailing commas without rewriting quoted values.""" + parts: list[str] = [] + i = 0 + comma: int | None = None + while i < len(text): + fragment = text[i] + if text.startswith("//", i): + end = text.find("\n", i + 2) + i = len(text) if end == -1 else end + parts.append(" ") + continue + if text.startswith("/*", i): + end = text.find("*/", i + 2) + if end == -1: + raise ValueError("unterminated comment") + i = end + 2 + parts.append(" ") + continue + if fragment == '"': + start = i + i += 1 + while i < len(text) and text[i] != '"': + i += 2 if text[i] == "\\" else 1 + if i >= len(text): + raise ValueError("unterminated string") + fragment = text[start:i + 1] + if not fragment.isspace(): + if comma is not None and fragment in ("}", "]"): + parts[comma] = "" + comma = len(parts) if fragment == "," else None + parts.append(fragment) + i += 1 + return "".join(parts) + + +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError("ambiguous duplicate key") + value[key] = item + return value + + +def _linked(info: os.stat_result) -> bool: + # Windows junctions also use reparse points. Treat the entire category as + # outside the scan rather than following vendor-specific redirections. + return stat.S_ISLNK(info.st_mode) or bool( + getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ) + + +def _manifest_type(relative: Path) -> str | None: + name = relative.as_posix() + if os.name == "nt": + name = name.lower() + parts = name.split("/") + if parts[-1] == "package.json": + return "npm" + if len(parts) == 3 and parts[0] == ".devcontainer" and parts[-1] == "devcontainer.json": + return "devcontainer" + return MANIFEST_TYPES.get(name) + + +def _manifests(root: Path, report: PreflightReport) -> Iterator[tuple[Path, str]]: + pending = [Path(".")] + remaining = MAX_ENTRIES + manifests = 0 + while pending: + directory = pending.pop() + try: + with os.scandir(root / directory) as entries: + batch = list(islice(entries, remaining + 1)) + except OSError: + report.note(directory, "directory_unreadable") + continue + if len(batch) > remaining: + report.note(directory, "repository_entry_limit_reached") + return + remaining -= len(batch) + for entry in sorted(batch, key=lambda x: x.name): + relative = directory / entry.name + if entry.name in SKIP_DIRS: + continue + try: + info = entry.stat(follow_symlinks=False) + except OSError: + report.note(relative, "entry_unreadable") + continue + if _linked(info): + report.note(relative, "linked_path_skipped") + elif stat.S_ISDIR(info.st_mode): + if len(relative.parts) <= MAX_DEPTH: + pending.append(relative) + else: + report.note(relative, "directory_depth_limit_reached") + elif kind := _manifest_type(relative): + if not stat.S_ISREG(info.st_mode): + report.note(relative, "non_regular_manifest") + continue + if manifests == MAX_MANIFESTS: + report.note(relative, "manifest_limit_reached") + return + manifests += 1 + yield relative, kind + + +def _words(command: object) -> list[str] | None: + if isinstance(command, list) and command and all(isinstance(x, str) and x for x in command): + return command + if not isinstance(command, str) or any(char in command for char in "\r\n;&|<>`"): + return None + try: + lexer = shlex.shlex(command, posix=True) + lexer.whitespace_split = True + lexer.commenters = lexer.escape = "" + return list(lexer) + except ValueError: + return None + + +def _reference( + root: Path, cwd: Path | None, command: object, root_markers: tuple[str, ...], +) -> Reference: + """Check the path of a simple declared script without reading it.""" + words = _words(command) + if not words: + return Reference("not_resolved") + first = words[0].replace("\\", "/") + if first.lower().removesuffix(".exe") in INTERPRETERS and len(words) > 1: + path = words[1].replace("\\", "/") + if path.startswith("-"): + return Reference("not_resolved") + elif "/" in first or PurePosixPath(first).suffix.lower() in SCRIPT_SUFFIXES: + path = first + else: + return Reference("not_identified") + # Recognize documented root markers only as a prefix. No arbitrary + # variable expansion, and a shell string with single quotes stays opaque. + if not isinstance(command, str) or "'" not in command: + for marker in root_markers: + if path.startswith(marker + "/"): + path = path[len(marker) + 1:] + cwd = Path(".") + break + if not SCRIPT_PATH.fullmatch(path) or path.startswith("-"): + return Reference("not_resolved") + candidate = PurePosixPath(path) + if candidate.is_absolute() or PureWindowsPath(path).drive or ".." in candidate.parts: + return Reference("outside_or_parent_path") + if not candidate.parts or cwd is None or ":" in path: + return Reference("not_resolved") + relative = cwd / Path(*candidate.parts) + current = root + try: + for part in relative.parts: + current = current / part + info = current.lstat() + if _linked(info): + return Reference("linked_path_skipped") + except FileNotFoundError: + return Reference("not_found") + except OSError: + return Reference("unreadable") + if not stat.S_ISREG(info.st_mode): + return Reference("not_regular") + if info.st_size > MAX_FILE_BYTES: + return Reference("oversized", relative.as_posix(), info.st_size) + return Reference("present_uninspected", relative.as_posix(), info.st_size) + + +def _object(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError("unsupported manifest field") + return value + + +def _array(value: object) -> list[Any]: + if not isinstance(value, list): + raise ValueError("unsupported manifest field") + return value + + +def _mcp_entries(doc: dict[str, Any], vscode: bool) -> Iterator[tuple[str, str, str, object, Path | None]]: + key = "servers" if vscode else "mcpServers" + for i, entry in enumerate(_object(doc.get(key, {})).values(), 1): + record = _object(entry) + marker = f"{key} entry {i}" + if isinstance(record.get("command"), str) and record["command"]: + args = _array(record.get("args", [])) + # MCP clients do not share a cwd contract. Inventory the process + # without claiming a relative argument resolves at the repo root. + yield marker, "mcp_server", "process", [record["command"], *args], None + elif isinstance(record.get("url"), str) and record["url"]: + yield marker, "mcp_server", "remote", None, None + else: + yield marker, "mcp_server", "unresolved", None, None + if "headersHelper" in record: + yield marker + " headersHelper", "mcp_headers_helper", "shell", record["headersHelper"], None + + +def _entries( + relative: Path, kind: str, doc: dict[str, Any], +) -> Iterator[tuple[str, str, str, object, Path | None]]: + if kind == "claude": + hooks = _object(doc.get("hooks", {})) + for i, group in enumerate(_array(hooks.get("SessionStart", [])), 1): + for j, hook in enumerate(_array(_object(group).get("hooks", [])), 1): + handler = _object(hook) + mode = handler.get("type") + launch = {"command": "shell", "http": "remote", "prompt": "agent", "agent": "agent"} + command = handler.get("command") if mode == "command" else None + if isinstance(command, str) and "args" in handler: + command = [command, *_array(handler["args"])] + launch["command"] = "process" + yield (f"hooks.SessionStart group {i} handler {j}", "claude_session_start", + launch.get(mode, "unresolved") if isinstance(mode, str) else "unresolved", + command, Path(".")) + elif kind in ("mcp", "vscode_mcp"): + yield from _mcp_entries(doc, kind == "vscode_mcp") + elif kind == "tasks": + for i, task in enumerate(_array(doc.get("tasks", [])), 1): + task = _object(task) + if _object(task.get("runOptions", {})).get("runOn") == "folderOpen": + mode = task.get("type") + mechanism = mode if mode in ("process", "shell") else "unresolved" + bracket = f"tasks entry {i}" + command = task.get("command") + if mechanism == "process" and isinstance(command, str) and "args" in task: + command = [command, *_array(task["args"])] + workspace_default = "cwd" not in _object(task.get("options", {})) + yield bracket, "vscode_folder_open", mechanism, command, Path(".") if workspace_default else None + if task.get("dependsOn"): + yield bracket + " dependsOn", "vscode_folder_open", "dependency", None, None + elif kind == "npm": + scripts = _object(doc.get("scripts", {})) + for event in NPM_INSTALL_EVENTS: + if event in scripts: + command = scripts[event] + if not isinstance(command, str): + raise ValueError("npm lifecycle script is not a string") + yield f"scripts.{event}", "npm_install", "shell", command, relative.parent + else: + for event in CONTAINER_EVENTS: + if event in doc: + # Commands after initialization run in a container with its + # own paths. Parallel command objects stay opaque at this layer. + yield event, "devcontainer_lifecycle", ( + "host" if event == "initializeCommand" else "container" + ), doc[event], None + + +def _read_manifest(path: Path, limit: int) -> bytes | None: + info = path.lstat() + if _linked(info) or not stat.S_ISREG(info.st_mode): + raise OSError("manifest is not a regular file") + if info.st_size > limit: + return None + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + with os.fdopen(os.open(path, flags), "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise OSError("manifest is not a regular file") + data = stream.read(limit + 1) + return data if len(data) <= limit else None + + +def inspect_repository(target: Path) -> PreflightReport: + report = PreflightReport(str(target)) + try: + root_info = target.lstat() + except OSError: + report.note(Path("."), "repository_unreadable") + return report + if _linked(root_info) or not stat.S_ISDIR(root_info.st_mode): + report.note(Path("."), "expected_a_regular_directory") + return report + + remaining = MAX_TOTAL_BYTES + for relative, kind in _manifests(target, report): + path = target / relative + limit = min(MAX_FILE_BYTES, remaining) + try: + raw = _read_manifest(path, limit) + if raw is None: + report.note(relative, "manifest_size_limit" if limit == MAX_FILE_BYTES else "total_byte_limit") + if limit != MAX_FILE_BYTES: + return report + continue + remaining -= len(raw) + text = raw.decode("utf-8-sig") + if kind in ("tasks", "vscode_mcp", "devcontainer"): + text = _jsonc(text) + document = _object(json.loads(text, object_pairs_hook=_unique_object)) + markers = ("${CLAUDE_PROJECT_DIR}",) if kind == "claude" else () + if kind in ("tasks", "vscode_mcp") or relative.as_posix().lower() == ".cursor/mcp.json": + markers = ("${workspaceFolder}",) + for location, trigger, mechanism, command, cwd in _entries(relative, kind, document): + if len(report.surfaces) == MAX_SURFACES: + report.note(relative, "surface_limit_reached") + return report + reference = (Reference("remote_uninspected") if mechanism == "remote" + else _reference(target, cwd, command, markers)) + report.surfaces.append(Surface(relative.as_posix(), location, trigger, mechanism, reference)) + report.manifests_read += 1 + except OSError: + report.note(relative, "manifest_unreadable") + except (ValueError, RecursionError): + report.note(relative, "invalid_or_unsupported_manifest") + return report diff --git a/apps/orchestrator/tests/test_repository_preflight.py b/apps/orchestrator/tests/test_repository_preflight.py new file mode 100644 index 0000000..f9f1ab7 --- /dev/null +++ b/apps/orchestrator/tests/test_repository_preflight.py @@ -0,0 +1,167 @@ +"""Synthetic declarations are inspected as data and never executed.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from agentmetry.core.scanner import preflight + + +def _write(root: Path, name: str, content: object) -> Path: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content if isinstance(content, str) else json.dumps(content), encoding="utf-8") + return path + + +def test_preflight_inventories_the_documented_triggers(tmp_path): + script = _write(tmp_path, "scripts/setup.js", "synthetic script contents") + _write(tmp_path, ".claude/settings.json", {"hooks": { + "SessionStart": [{"hooks": [ + {"type": "command", "command": 'node "${CLAUDE_PROJECT_DIR}/scripts/setup.js"'}, + {"type": "command", "command": "node scripts/setup.js"}, + ]}], + "PreToolUse": [{"hooks": [{"type": "command", "command": "manual-control"}]}], + }}) + _write(tmp_path, ".mcp.json", {"mcpServers": { + "synthetic-private-label": {"type": "http", "url": "https://example.invalid/synthetic-private-url", + "headers": {"Authorization": "synthetic-private-value"}}, + }}) + _write(tmp_path, ".cursor/mcp.json", {"mcpServers": { + "synthetic-private-label": {"command": "node", "args": ["${workspaceFolder}/scripts/setup.js"]}, + }}) + _write(tmp_path, ".vscode/tasks.json", ''' + { + // Folder-open tasks use JSONC. + "tasks": [ + {"label":"synthetic-private-label","type":"process","command":"node", + "args":["${workspaceFolder}/scripts/setup.js"],"runOptions":{"runOn":"folderOpen"},}, + {"label":"manual", "type":"shell","command":"manual-control"}, + ], + } + ''') + _write(tmp_path, ".devcontainer/example/devcontainer.json", { + "initializeCommand": ["echo", "synthetic-private-value"], + "postCreateCommand": {"one": "echo synthetic-private-value"}, + }) + _write(tmp_path, "package.json", {"scripts": {"postinstall": "node ./scripts/setup.js", "test": "manual-control"}}) + _write(tmp_path, "packages/example/package.json", {"scripts": {"prepare": "echo synthetic-private-value"}}) + _write(tmp_path, "node_modules/dependency/package.json", {"scripts": {"postinstall": "unused"}}) + + report = preflight.inspect_repository(tmp_path) + assert report.exit_code() == 0 + assert report.manifests_read == 7 + assert {surface.trigger for surface in report.surfaces} == { + "claude_session_start", "mcp_server", "vscode_folder_open", "devcontainer_lifecycle", "npm_install", + } + assert len(report.surfaces) == 9 + local = [row for row in report.surfaces if row.reference.status == "present_uninspected"] + assert len(local) == 5 + assert all(row.reference.path == "scripts/setup.js" and row.reference.size_bytes == script.stat().st_size + for row in local) + remote = [row for row in report.surfaces if row.mechanism == "remote"] + assert len(remote) == 1 and remote[0].reference.status == "remote_uninspected" + assert {row.mechanism for row in report.surfaces if row.trigger == "devcontainer_lifecycle"} == {"host", "container"} + for output in (json.dumps(report.to_json()), "\n".join(report.lines())): + assert "synthetic-private" not in output and "manual-control" not in output + assert "synthetic script contents" not in output + + +def test_preflight_separates_unknown_references_from_declared_paths(tmp_path, monkeypatch): + monkeypatch.setattr(preflight, "MAX_FILE_BYTES", 260) + large = _write(tmp_path, "large.js", "x" * 261) + _write(tmp_path, "package.json", {"scripts": { + "preinstall": "node ./missing.js", "install": "node ./large.js", + "postinstall": "node ../private.js", "prepare": "node ${UNKNOWN_DIR}/example.js", + "postprepare": "node example.js && echo example", + }}) + report = preflight.inspect_repository(tmp_path) + refs = {row.location: row.reference for row in report.surfaces} + assert report.exit_code() == 0 + assert refs["scripts.preinstall"] == preflight.Reference("not_found") + assert refs["scripts.install"] == preflight.Reference("oversized", "large.js", large.stat().st_size) + assert refs["scripts.postinstall"] == preflight.Reference("outside_or_parent_path") + assert refs["scripts.prepare"] == refs["scripts.postprepare"] == preflight.Reference("not_resolved") + + +def test_preflight_resolves_a_nested_package_relative_to_its_directory(tmp_path): + _write(tmp_path, "packages/example/task.py", "synthetic") + _write(tmp_path, "packages/example/package.json", {"scripts": {"postinstall": "python task.py"}}) + report = preflight.inspect_repository(tmp_path) + (surface,) = report.surfaces + assert surface.reference.path == "packages/example/task.py" + + +def test_preflight_jsonc_preserves_quoted_comment_and_comma_sequences(): + source = r'{"value":"https://example.invalid/,}/*quoted*/\"x", /* note */ "array":[1,],}' + assert json.loads(preflight._jsonc(source)) == { + "value": 'https://example.invalid/,}/*quoted*/"x', "array": [1], + } + + +def test_preflight_reports_invalid_duplicate_and_oversized_manifests(tmp_path, monkeypatch): + monkeypatch.setattr(preflight, "MAX_FILE_BYTES", 100) + _write(tmp_path, ".claude/settings.json", '{"hooks":{},"hooks":{}}') + _write(tmp_path, ".vscode/tasks.json", '{"tasks":[}') + _write(tmp_path, "large/package.json", " " * 101) + _write(tmp_path, "package.json", {"scripts": {"postinstall": "echo example"}}) + report = preflight.inspect_repository(tmp_path) + assert report.exit_code() == 2 + assert report.to_json()["coverage"] == "partial" + assert len(report.surfaces) == 1 + assert {x["path"]: x["reason"] for x in report.unobserved} == { + ".claude/settings.json": "invalid_or_unsupported_manifest", + ".vscode/tasks.json": "invalid_or_unsupported_manifest", + "large/package.json": "manifest_size_limit", + } + + +def test_preflight_skips_linked_paths_and_script_references(tmp_path): + repo = tmp_path / "repo" + outside = _write(tmp_path, "outside/package.json", {"scripts": {"postinstall": "outside-control"}}) + _write(repo, "package.json", {"scripts": {"postinstall": "node ./linked.js"}}) + try: + (repo / "linked.js").symlink_to(outside) + (repo / "outside").symlink_to(outside.parent, target_is_directory=True) + except (NotImplementedError, OSError): + pytest.skip("symlink creation is unavailable") + report = preflight.inspect_repository(repo) + assert report.exit_code() == 2 + assert report.manifests_read == 1 + assert {x["path"] for x in report.unobserved} == {"linked.js", "outside"} + assert [row.reference.status for row in report.surfaces] == ["linked_path_skipped"] + + +@pytest.mark.parametrize( + "limit, value, reason", + [ + ("MAX_ENTRIES", 1, "repository_entry_limit_reached"), + ("MAX_MANIFESTS", 1, "manifest_limit_reached"), + ("MAX_TOTAL_BYTES", 1, "total_byte_limit"), + ("MAX_SURFACES", 1, "surface_limit_reached"), + ], +) +def test_preflight_reports_reached_limits(tmp_path, monkeypatch, limit, value, reason): + monkeypatch.setattr(preflight, limit, value) + content = {"scripts": {"preinstall": "echo test", "postinstall": "echo test"}} + _write(tmp_path, "package.json", content) + _write(tmp_path, "nested/package.json", content) + report = preflight.inspect_repository(tmp_path) + assert report.exit_code() == 2 + assert any(x["reason"] == reason for x in report.unobserved) + + +def test_preflight_cli_emits_json_and_preserves_the_existing_scanner(tmp_path, capsys): + from agentmetry.cli import main + + _write(tmp_path, "package.json", {"scripts": {"postinstall": "echo test"}}) + assert main(["scan", "--preflight", str(tmp_path), "--json"]) == 0 + current = json.loads(capsys.readouterr().out) + assert current["mode"] == "repository_preflight" and len(current["surfaces"]) == 1 + assert main(["scan", "pypi:pydantic-settings", "--json"]) == 0 + legacy = json.loads(capsys.readouterr().out) + assert "mode" not in legacy and legacy["verdict"] == "no findings" + assert main(["scan", "--preflight", str(tmp_path), "npm:example"]) == 2 diff --git a/docs/scanner.md b/docs/scanner.md index 4b76991..e0ce855 100644 --- a/docs/scanner.md +++ b/docs/scanner.md @@ -10,6 +10,7 @@ a server and never fetches a package. ```bash agentmetry scan # MCP servers, and instruction files in Downloads, Desktop and ~ agentmetry scan ./skills # a file or folder: skills, rules, a saved tools/list +agentmetry scan --preflight ./repo # inventory declarations that can start execution agentmetry scan pypi:request # one package name agentmetry scan --json # the report as JSON agentmetry scan --known names.txt # add your own package names (npm:/pypi: or bare) @@ -23,7 +24,43 @@ install, so a finding stops the script that ran it. The output never contains the word "safe". "No findings" means these checks found nothing, which is all it can honestly mean. -## The two checks +## Repository execution preflight + +Run `agentmetry scan --preflight PATH` against a checkout before opening it +in an agent or running project setup. Add `--json` for structured inventory. +The preflight reads manifests in the named directory and reports where each +declaration is defined. Entries describe potential execution and carry no +malware verdict or severity. + +| Manifest | Declaration | +|---|---| +| `.claude/settings.json`, `.claude/settings.local.json` | Claude Code `SessionStart` handlers. [Hooks reference](https://code.claude.com/docs/en/hooks#sessionstart) | +| `.mcp.json`, `.cursor/mcp.json`, `.vscode/mcp.json` | Project MCP process or remote server declarations and any declared header helper. [Claude Code MCP](https://code.claude.com/docs/en/mcp), [Cursor MCP](https://cursor.com/docs/mcp), [VS Code MCP](https://code.visualstudio.com/docs/agent-customization/mcp-servers) | +| `.vscode/tasks.json` | Tasks with `runOptions.runOn: folderOpen`. A dependency entry signals that further tasks need review. [VS Code tasks](https://code.visualstudio.com/docs/debugtest/tasks#_run-behavior) | +| `package.json` in the checkout | npm install lifecycle entries. Scripts such as `test` are outside this inventory. [npm lifecycle order](https://docs.npmjs.com/cli/v11/using-npm/scripts/#life-cycle-operation-order) | +| `.devcontainer.json`, `.devcontainer/devcontainer.json`, `.devcontainer/*/devcontainer.json` | Host initialization and container creation, start and attach lifecycle fields. [Dev Container specification](https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-reference.md) | + +The report keeps manifest paths, entry locations and reference status. Commands, +arguments, environment fields, server names, endpoint URLs and script contents +stay out of the report. A simple npm script path can be checked relative to its +package. For simple startup hooks and default editor tasks, relative paths +are checked from the repository root. That assumes you open this directory as +the project. Supported project-root placeholders can also locate a candidate +script in the checkout. Other shell syntax, client-dependent working directories, +task dependencies and container paths remain unresolved. A present script is +uninspected. The preflight does not resolve agent trust, policy or enablement. + +The scan skips symbolic links and Windows junctions, dependency and build +directories, and manifests outside the supported paths. It caps directory +entries at 12,000, depth at 32, manifests at 512, data at 1,000,000 bytes per +manifest and 16,000,000 bytes total, and declarations at 1,000. Malformed or +unreadable manifests, links and limits are reported as partial coverage. +Preflight exits **0** when it reads the supported manifests in scope or finds +none, and **2** on an incomplete inventory. Script reference statuses describe +what could be located and do not change manifest coverage. This inventory cannot +see hooks from an installed plugin, downloaded dependencies or child processes. + +## Existing scan checks ### Text that instructs the model (#188)