diff --git a/eval/dataset/REVIEWBOT_EVAL.md b/eval/dataset/REVIEWBOT_EVAL.md index fe0cd4ab9..957205889 100644 --- a/eval/dataset/REVIEWBOT_EVAL.md +++ b/eval/dataset/REVIEWBOT_EVAL.md @@ -36,6 +36,31 @@ ONLY_ITEMS=4893 GEN_REPLICATES=1 bash eval/dataset/run_reviewbot_monthly.sh smok A non-month tag writes `results/reviewbot/REVIEWBOT_smoke.md` and never touches the month-over-month `INDEX.md`. +The 15 frozen train+val PRs are now merged. The monthly command explicitly +uses `REVIEWBOT_EVAL_ARCHIVED_REPLAY=1`: an evaluation-only adapter presents +their pinned historical open snapshot to the installed ReviewBot Direct +pipeline. It requires `POST_MODE=shadow`, `REVIEW_CONTEXT_MODE=no_discussion`, +and `post=False`, and rejects every non-GET GitHub request. ReviewBot's +production rule remains open PRs only (drafts require an explicit request). +For a future live open-PR dataset, invoke `run_reviewbot_arm.py` directly +without the replay flag. + +`goal-eval/expected_pr_bases.json` pins each historical merge base alongside +the existing head pins. The adapter reads the frozen `gt/pr.diff` for file +metadata; GitHub's current file list has drifted for PRs 4816, 4926, and +4977. Before changing any pin, verify all 15 patches against a local +vllm-omni Git checkout: + +```bash +python3 eval/dataset/verify_reviewbot_archived_bases.py --repo /path/to/vllm-omni +``` + +The verifier ignores only Git's display-only object IDs and optional hunk +function names. Each normalized historical `base..head` patch must exactly +equal its frozen ground-truth diff. The base-pin file, selected frozen diffs, +adapter SHA-256s, and replay mode are recorded in each arm manifest, so a +resumed campaign cannot silently mix historical snapshots or adapter versions. + `--dry-run` is deliberately planning-only: it does not require the release interpreter or manifest and does not probe installed packages. @@ -80,8 +105,10 @@ interpreter or manifest and does not probe installed packages. match them exactly. This strict envelope applies only to `reviewbot_YYYY-MM`; non-month smoke tags retain legacy-manifest compatibility. - Child processes receive no `PYTHONPATH`, `PYTHONHOME`, provider source-path - override, or user site. The harness invokes only the selected installed - `python -m omni_reviewbot`; it never imports a sibling ReviewBot checkout. + override, or user site. The harness runs the selected installed ReviewBot + wheel through the evaluation-only archived adapter for these merged PRs; + direct live-PR runs use `python -m omni_reviewbot`. It never imports a + sibling ReviewBot checkout. - `POST_MODE=shadow` asserted from CLI output — nothing can reach GitHub. - `REVIEW_CONTEXT_MODE=no_discussion` asserted (`review_context: no_discussion (0 threads)`): on these PRs the historical review diff --git a/eval/dataset/build_reviewbot_report.py b/eval/dataset/build_reviewbot_report.py index 53cfe1c95..b3be5d545 100755 --- a/eval/dataset/build_reviewbot_report.py +++ b/eval/dataset/build_reviewbot_report.py @@ -71,6 +71,9 @@ "started_at", } _CONFIG_FIELDS = {"release", "env", "review_context_mode", "post_mode"} +_ARCHIVED_REPLAY_FIELDS = { + "mode", "bases_sha256", "diffs_sha256", "adapter_sha256", +} _ENV_FIELDS = { "AGENT_PROVIDER", "REVIEW_MODEL", @@ -195,8 +198,19 @@ def _monthly_provenance_problems( if not isinstance(config, dict): problems.append("config is not an object") return problems - if set(config) != _CONFIG_FIELDS: + if set(config) not in (_CONFIG_FIELDS, _CONFIG_FIELDS | {"archived_replay"}): problems.append("config fields do not match the runner config shape") + if "archived_replay" in config: + replay = config["archived_replay"] + if not isinstance(replay, dict) or set(replay) != _ARCHIVED_REPLAY_FIELDS: + problems.append("config.archived_replay fields do not match") + else: + if replay.get("mode") != "frozen_open_pr_v1": + problems.append("config.archived_replay.mode is invalid") + for name in ("bases_sha256", "diffs_sha256", "adapter_sha256"): + digest = replay.get(name) + if not isinstance(digest, str) or not SHA256_FINGERPRINT.fullmatch(digest): + problems.append(f"config.archived_replay.{name} is invalid") if config.get("post_mode") != "shadow": problems.append("config.post_mode must be 'shadow'") if config.get("review_context_mode") != "no_discussion": @@ -520,6 +534,14 @@ def render(tag, aggregate, item_rows, wins, gap_rows, config, "| dim | Δ | 95% CI | items +/− | n |", "|---|---|---|---|---|", ] + replay = config.get("archived_replay") + if replay: + lines[6:6] = [ + ( + "- Dataset: frozen merged-PR shadow replay, historical " + f"base pins `{replay['bases_sha256']}`" + ), + ] for dim in DIMS: a = aggregate[dim] lines.append( diff --git a/eval/dataset/goal-eval/expected_pr_bases.json b/eval/dataset/goal-eval/expected_pr_bases.json new file mode 100644 index 000000000..ea0c6e191 --- /dev/null +++ b/eval/dataset/goal-eval/expected_pr_bases.json @@ -0,0 +1,17 @@ +{ + "4804": "a25636d7803a2a5da22cb5beae2363eef1f09fad", + "4810": "b443504d619b4a66bfcb5e6f5900e91458dbe11d", + "4816": "a560ed184d9197855f1cccf46cb1cac87b0a7138", + "4817": "e7b281799ec80cabc1493f3c8a623589a95cf1de", + "4825": "d526dc93e89aa4d806a9111aab496b0bc04c8937", + "4837": "431a7c71ac15fc824edfe9dd59c10f7566bde2ed", + "4859": "a8b2bf11d2f2e9282e2182c2aad932d58777e3dc", + "4870": "0798b0bf03cb6e7c4dc40ee58fe12ce492ce255f", + "4893": "ddba6de2ab658a0fdb6f3f72cac9d2c3e2c19aea", + "4923": "ca0ae7269ca3e9487645cf66088fdfc338951da9", + "4926": "5c51782c6bec3c7976c528e5899e89452278c448", + "4950": "4908b85731731da46c4560cbe2b567c94fb032a7", + "4970": "da15a4cb1cfeae3ca02bc075bbe64fe64de59820", + "4977": "b7d08fe5f6b1306eee44172c6589e3225de6897e", + "5009": "cbd3939c3d5099a805397f0cc29a41aa3e9c3235" +} diff --git a/eval/dataset/reviewbot_archived_replay.py b/eval/dataset/reviewbot_archived_replay.py new file mode 100644 index 000000000..42cd5959e --- /dev/null +++ b/eval/dataset/reviewbot_archived_replay.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Replay one frozen, now-merged PR through an installed ReviewBot release. + +This evaluation-only adapter supplies the historical open snapshot and the +frozen file list to the normal Direct pipeline. The GitHub transport is +read-only, and the pipeline is always called with post=False. Production PR +eligibility and publication code are neither imported from source nor changed. +""" +from __future__ import annotations + +import argparse +import os +import re +from dataclasses import replace +from pathlib import Path + +SHA = re.compile(r"[0-9a-f]{40}\Z") +FILE_HEADER = re.compile(r"diff --git a/(.+) b/(.+)\Z") + + +def frozen_files(diff: str) -> list[dict[str, object]]: + """Project the checked-in Git patch into ReviewBot's ChangedFile fields.""" + files: list[dict[str, object]] = [] + current: dict[str, object] | None = None + patch: list[str] = [] + in_hunk = False + + def finish() -> None: + if current is not None: + current["patch"] = "\n".join(patch) + "\n" if patch else None + files.append(current) + + for line in diff.splitlines(): + if line.startswith("diff --git "): + finish() + match = FILE_HEADER.fullmatch(line) + if match is None or match.group(1) != match.group(2): + raise ValueError(f"unsupported frozen file header: {line}") + current = { + "path": match.group(2), "status": "modified", + "additions": 0, "deletions": 0, + } + patch = [] + in_hunk = False + elif current is None: + raise ValueError("frozen diff has content before its first file") + elif line.startswith("new file mode "): + current["status"] = "added" + elif line.startswith("deleted file mode "): + current["status"] = "removed" + elif line.startswith("rename from ") or line.startswith("copy from "): + raise ValueError("frozen diff rename/copy needs explicit projection") + elif line.startswith("Binary files ") or line == "GIT binary patch": + raise ValueError("binary frozen diffs need explicit projection") + elif line.startswith("@@ "): + in_hunk = True + patch.append(line) + elif in_hunk: + patch.append(line) + if line.startswith("+"): + current["additions"] = int(current["additions"]) + 1 + elif line.startswith("-"): + current["deletions"] = int(current["deletions"]) + 1 + finish() + if not files: + raise ValueError("frozen diff contains no files") + paths = [str(item["path"]) for item in files] + if len(paths) != len(set(paths)): + raise ValueError("frozen diff repeats a file") + return files + + +def historical_pull(pull, *, pr: int, head: str, base: str): + """Verify the live archived identity before presenting the frozen view.""" + if pull.number != pr or pull.head_sha != head: + raise ValueError(f"pr{pr} no longer has the pinned head") + if pull.state != "closed" or not pull.merged_at: + raise ValueError(f"pr{pr} is not a merged archived PR") + return replace( + pull, state="open", base_sha=base, merged_at="", merge_commit_sha="", + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pr", type=int, required=True) + parser.add_argument("--expected-head", required=True) + parser.add_argument("--base", required=True) + parser.add_argument("--diff", type=Path, required=True) + args = parser.parse_args() + if os.environ.get("REVIEWBOT_EVAL_ARCHIVED_REPLAY") != "1": + parser.error("archived replay requires REVIEWBOT_EVAL_ARCHIVED_REPLAY=1") + if os.environ.get("POST_MODE") != "shadow": + parser.error("archived replay requires POST_MODE=shadow") + if os.environ.get("REVIEW_CONTEXT_MODE") != "no_discussion": + parser.error("archived replay requires REVIEW_CONTEXT_MODE=no_discussion") + if args.pr <= 0 or not SHA.fullmatch(args.expected_head) or not SHA.fullmatch(args.base): + parser.error("invalid PR number or pinned commit SHA") + if args.diff.name != f"pr{args.pr}.diff" or not args.diff.is_file(): + parser.error("diff must be the frozen patch for the selected PR") + + # Import only the paired wheel installed in the runner's private venv. + from omni_reviewbot.cli import build_runtime + from omni_reviewbot.config import Settings + from omni_reviewbot.models import ChangedFile + + settings = Settings.from_env() + if settings.post or settings.review_context_mode != "no_discussion": + parser.error("installed ReviewBot settings violate shadow replay") + diff_files = [ + ChangedFile(**fields) + for fields in frozen_files(args.diff.read_text(encoding="utf-8")) + ] + runtime = build_runtime(settings, recover_running=False) + github = runtime.github + request = github._request + + def read_only(method, path, payload=None): + if method != "GET" or payload is not None: + raise RuntimeError("archived evaluation rejects GitHub writes") + return request(method, path, payload) + + github._request = read_only + original_get_pull = github.get_pull + + def get_pull(number): + if number != args.pr: + raise RuntimeError("archived evaluation rejects another PR") + return historical_pull( + original_get_pull(number), + pr=args.pr, head=args.expected_head, base=args.base, + ) + + def list_files(number): + if number != args.pr: + raise RuntimeError("archived evaluation rejects another PR") + return list(diff_files) + + github.get_pull = get_pull + github.list_files = list_files + outcome = runtime.pipeline.run( + pr_number=args.pr, expected_head=args.expected_head, post=False, + ) + print(outcome.body) + print(f"status: {outcome.status}") + metrics = outcome.metrics or {} + if "review_context_mode" in metrics: + print( + f"review_context: {metrics['review_context_mode']} " + f"({metrics['review_context_count']} threads)" + ) + if "changed_files" in metrics: + print(f"changed_files: {metrics['changed_files']}") + if outcome.artifact_path: + print(f"artifact: {outcome.artifact_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/dataset/run_reviewbot_arm.py b/eval/dataset/run_reviewbot_arm.py index 6e6131470..02efb995e 100755 --- a/eval/dataset/run_reviewbot_arm.py +++ b/eval/dataset/run_reviewbot_arm.py @@ -29,6 +29,7 @@ venv) · REVIEWBOT_ENV_FILE (optional; loaded first, hard overrides win) · REVIEWBOT_TIMEOUT_S=1800 · REVIEWBOT_EVAL_ROOT (tests only) · REVIEWBOT_EVAL_ALLOW_UNPAIRED=1 (non-month throwaway runs only) +REVIEWBOT_EVAL_ARCHIVED_REPLAY=1 (frozen, now-merged dataset PRs) Flags: --dry-run (print the plan, touch nothing) · --preflight (doctor only) """ from __future__ import annotations @@ -54,6 +55,8 @@ HERE = Path(os.environ.get("REVIEWBOT_EVAL_ROOT") or Path(__file__).parent) DATASET = HERE / "vllm_omni_dataset.yaml" EXPECTED_HEADS = HERE / "goal-eval" / "expected_pr_heads.json" +EXPECTED_BASES = HERE / "goal-eval" / "expected_pr_bases.json" +ARCHIVED_REPLAY = HERE / "reviewbot_archived_replay.py" GT = HERE / "gt" ARMS = HERE / "arms" STATE_DIR = HERE.parent / "raw" / "reviewbot_state" @@ -140,14 +143,29 @@ class ReleaseValidationError(RuntimeError): """The selected interpreter is not the manifest's paired artifact.""" -def build_config(child_env: dict[str, str], release: dict[str, Any]) -> dict: +def build_config( + child_env: dict[str, str], release: dict[str, Any], *, + archived_bases_sha256: str | None = None, + archived_diffs_sha256: str | None = None, + archived_adapter_sha256: str | None = None, +) -> dict: """Stable arm identity from paired artifacts and behavior-only settings.""" - return { + config = { "release": release, "review_context_mode": "no_discussion", "post_mode": "shadow", "env": {key: child_env.get(key, "") for key in _BEHAVIOR_KEYS}, } + if archived_bases_sha256 is not None: + if archived_diffs_sha256 is None or archived_adapter_sha256 is None: + raise ValueError("archived replay requires all source fingerprints") + config["archived_replay"] = { + "mode": "frozen_open_pr_v1", + "bases_sha256": "sha256:" + archived_bases_sha256, + "diffs_sha256": "sha256:" + archived_diffs_sha256, + "adapter_sha256": "sha256:" + archived_adapter_sha256, + } + return config def _canonical_distribution(value: str) -> str: @@ -162,6 +180,16 @@ def _file_digest(path: Path) -> str: return digest.hexdigest() +def _frozen_diffs_digest(items: list[int]) -> str: + digest = hashlib.sha256() + for pr in items: + path = GT / f"pr{pr}.diff" + digest.update(path.name.encode("ascii") + b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + def _wheel_metadata(path: Path) -> tuple[str, str]: try: with zipfile.ZipFile(path) as archive: @@ -614,6 +642,9 @@ def __init__(self) -> None: self.allow_unpaired = ( os.environ.get("REVIEWBOT_EVAL_ALLOW_UNPAIRED") == "1" ) + self.archived_replay = ( + os.environ.get("REVIEWBOT_EVAL_ARCHIVED_REPLAY") == "1" + ) env_file_value = os.environ.get("REVIEWBOT_ENV_FILE", "").strip() env_file = Path(env_file_value).expanduser() if env_file_value else None # An explicitly selected env file loads first; the measurement contract @@ -637,10 +668,18 @@ def __init__(self) -> None: "PYTHONSAFEPATH": "1", } ) + if self.archived_replay: + child["REVIEWBOT_EVAL_ARCHIVED_REPLAY"] = "1" + else: + child.pop("REVIEWBOT_EVAL_ARCHIVED_REPLAY", None) self.child_env = child self.expected = { int(k): v for k, v in json.loads(EXPECTED_HEADS.read_text()).items() } + self.bases = ( + {int(k): v for k, v in json.loads(EXPECTED_BASES.read_text()).items()} + if self.archived_replay else {} + ) self.items = dataset_items() self.failures: list[str] = [] self._lock = threading.Lock() @@ -699,7 +738,18 @@ def ensure_config(self) -> dict: + os.pathsep + self.child_env.get("PATH", "") ) - self.config = build_config(self.child_env, release) + self.config = build_config( + self.child_env, release, + archived_bases_sha256=( + _file_digest(EXPECTED_BASES) if self.archived_replay else None + ), + archived_diffs_sha256=( + _frozen_diffs_digest(self.items) if self.archived_replay else None + ), + archived_adapter_sha256=( + _file_digest(ARCHIVED_REPLAY) if self.archived_replay else None + ), + ) return self.config def close(self) -> None: @@ -760,6 +810,16 @@ def _record_head(self, rep: int, stem: str, head: str) -> None: # --- invocation --- + def _review_command(self, pr: int) -> list[str]: + if self.archived_replay: + return [ + self.python, str(ARCHIVED_REPLAY), "--pr", str(pr), + "--expected-head", self.expected[pr], + "--base", self.bases[pr], + "--diff", str(GT / f"pr{pr}.diff"), + ] + return [self.python, "-m", "omni_reviewbot", "review", "--pr", str(pr)] + def preflight(self) -> None: config = self.ensure_config() if config["env"]["AGENT_PROVIDER"] == "cursor" and not os.environ.get( @@ -776,6 +836,15 @@ def preflight(self) -> None: sys.exit(f"pr{n} missing from {EXPECTED_HEADS}") if not (GT / f"pr{n}.diff").is_file(): sys.exit(f"gt/pr{n}.diff missing — cannot validate diff range") + if self.archived_replay and ( + n not in self.bases + or not isinstance(self.bases[n], str) + or not _SHA.fullmatch(self.bases[n]) + or not _SHA.fullmatch(self.expected[n]) + ): + sys.exit(f"pr{n} has no valid archived base/head pin") + if self.archived_replay and not ARCHIVED_REPLAY.is_file(): + sys.exit(f"archived replay script missing: {ARCHIVED_REPLAY}") STATE_DIR.mkdir(parents=True, exist_ok=True) doctor = subprocess.run( [self.python, "-m", "omni_reviewbot", "doctor"], @@ -794,7 +863,7 @@ def _invoke(self, pr: int, rep: int) -> None: if out_md.exists() and out_md.stat().st_size > 0: return completed = subprocess.run( - [self.python, "-m", "omni_reviewbot", "review", "--pr", str(pr)], + self._review_command(pr), cwd=STATE_DIR, env=self.child_env, capture_output=True, text=True, timeout=self.timeout, check=False, ) diff --git a/eval/dataset/run_reviewbot_monthly.sh b/eval/dataset/run_reviewbot_monthly.sh index dac671239..b8e9e9cfb 100755 --- a/eval/dataset/run_reviewbot_monthly.sh +++ b/eval/dataset/run_reviewbot_monthly.sh @@ -9,6 +9,9 @@ TAG="reviewbot_${MONTH}" GEN="${GEN_REPLICATES:-3}" cd "$(dirname "$0")" PY=${PYTHON:-python3} +# The frozen train+val dataset now consists of merged PRs. Replay their +# pinned historical open snapshots in the evaluation-only shadow adapter. +export REVIEWBOT_EVAL_ARCHIVED_REPLAY=1 echo "=== preflight ===" command -v claude >/dev/null || { diff --git a/eval/dataset/verify_reviewbot_archived_bases.py b/eval/dataset/verify_reviewbot_archived_bases.py new file mode 100644 index 000000000..b6b0825c4 --- /dev/null +++ b/eval/dataset/verify_reviewbot_archived_bases.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Prove that pinned historical bases reproduce every frozen review diff.""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +from pathlib import Path + +HERE = Path(__file__).resolve().parent +HUNK = re.compile(r"^(@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@).*$") + + +def normalized(diff: str) -> str: + """Ignore Git object IDs and optional hunk function names only.""" + lines = [] + for line in diff.splitlines(): + if line.startswith("index "): + continue + match = HUNK.fullmatch(line) + lines.append(match.group(1) if match else line) + return "\n".join(lines) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + args = parser.parse_args() + repo = args.repo.resolve() + if not (repo / ".git").exists(): + parser.error("--repo must be a local vllm-omni Git checkout") + heads = json.loads((HERE / "goal-eval/expected_pr_heads.json").read_text()) + bases = json.loads((HERE / "goal-eval/expected_pr_bases.json").read_text()) + if len(bases) != 15: + parser.error("expected exactly 15 train+val historical base pins") + for pr, base in sorted(bases.items(), key=lambda pair: int(pair[0])): + head = heads[pr] + actual = subprocess.run( + ["git", "-C", str(repo), "diff", "--no-ext-diff", "--binary", + "--no-color", f"{base}..{head}"], + capture_output=True, text=True, check=True, + ).stdout + expected = (HERE / "gt" / f"pr{pr}.diff").read_text() + if normalized(actual) != normalized(expected): + raise SystemExit(f"pr{pr}: pinned base/head diff differs from frozen GT") + print(f"pr{pr}: frozen diff verified") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/test_reviewbot_eval.py b/test/test_reviewbot_eval.py index 19a642866..7ddcbfca5 100644 --- a/test/test_reviewbot_eval.py +++ b/test/test_reviewbot_eval.py @@ -9,6 +9,7 @@ import subprocess import sys import zipfile +from dataclasses import replace from pathlib import Path import pytest @@ -18,6 +19,7 @@ report = importlib.import_module("build_reviewbot_report") arm = importlib.import_module("run_reviewbot_arm") +archive = importlib.import_module("reviewbot_archived_replay") TRAIN_VAL_PRS = [4804, 4810, 4816, 4817, 4825, 4837, 4859, 4870, 4893, @@ -50,6 +52,81 @@ def test_dataset_items_are_train_val_only_and_pinned(): assert all(n in expected for n in items) +def test_archived_base_pins_cover_frozen_train_val_dataset(): + bases = json.loads(arm.EXPECTED_BASES.read_text()) + assert set(bases) == {str(n) for n in TRAIN_VAL_PRS} + assert all(arm._SHA.fullmatch(sha) for sha in bases.values()) + for pr in TRAIN_VAL_PRS: + files = archive.frozen_files((arm.GT / f"pr{pr}.diff").read_text()) + assert len(files) == arm.gt_changed_files(pr) + assert all(file["path"] and file["patch"] for file in files) + + +def test_archived_file_projection_counts_only_hunk_lines(): + diff = ( + "diff --git a/a.py b/a.py\nindex 111..222 100644\n" + "--- a/a.py\n+++ b/a.py\n@@ -1 +1 @@\n-old\n+new\n" + "diff --git a/b.py b/b.py\nnew file mode 100644\n" + "--- /dev/null\n+++ b/b.py\n@@ -0,0 +1 @@\n+value\n" + ) + assert archive.frozen_files(diff) == [ + {"path": "a.py", "status": "modified", "additions": 1, + "deletions": 1, "patch": "@@ -1 +1 @@\n-old\n+new\n"}, + {"path": "b.py", "status": "added", "additions": 1, + "deletions": 0, "patch": "@@ -0,0 +1 @@\n+value\n"}, + ] + + +def test_archived_pull_requires_exact_merged_identity(): + # dataclasses.replace is used by the adapter, as it is for the installed + # ReviewBot PullRequest. A local frozen dataclass tests this without wheels. + from dataclasses import dataclass + + @dataclass(frozen=True) + class Pull: + number: int = 4893 + head_sha: str = "a" * 40 + base_sha: str = "b" * 40 + state: str = "closed" + merged_at: str = "2026-09-01" + merge_commit_sha: str = "c" * 40 + + live = Pull() + replay = archive.historical_pull(live, pr=4893, head="a" * 40, base="d" * 40) + assert (replay.state, replay.base_sha, replay.merged_at) == ("open", "d" * 40, "") + assert live.state == "closed" + for stale in (replace(live, head_sha="e" * 40), replace(live, state="open")): + with pytest.raises(ValueError): + archive.historical_pull(stale, pr=4893, head="a" * 40, base="d" * 40) + + +def test_archived_runner_identity_and_command_are_explicit(): + release = {"paired": True} + config = arm.build_config( + {}, release, archived_bases_sha256="a" * 64, + archived_diffs_sha256="b" * 64, + archived_adapter_sha256="c" * 64, + ) + assert config["archived_replay"] == { + "mode": "frozen_open_pr_v1", "bases_sha256": "sha256:" + "a" * 64, + "diffs_sha256": "sha256:" + "b" * 64, + "adapter_sha256": "sha256:" + "c" * 64, + } + assert "archived_replay" not in arm.build_config({}, release) + runner = arm.Runner.__new__(arm.Runner) + runner.archived_replay = True + runner.python = "/paired/.venv/bin/python" + runner.expected = {4893: "a" * 40} + runner.bases = {4893: "b" * 40} + command = runner._review_command(4893) + assert command[:2] == [runner.python, str(arm.ARCHIVED_REPLAY)] + assert command[-4:] == ["--base", "b" * 40, "--diff", str(arm.GT / "pr4893.diff")] + runner.archived_replay = False + assert runner._review_command(4893) == [ + runner.python, "-m", "omni_reviewbot", "review", "--pr", "4893", + ] + + def test_dry_run_plans_without_touching_anything(tmp_path): env = dict( os.environ, @@ -343,6 +420,29 @@ def test_monthly_behavior_modes_are_pinned( report.load_campaign("reviewbot_2026-09", 1, 2) +def test_monthly_accepts_archived_replay_provenance_and_rejects_drift(campaign_root): + _write_campaign(campaign_root) + + def add_replay(manifest): + manifest["config"]["archived_replay"] = { + "mode": "frozen_open_pr_v1", + "bases_sha256": "sha256:" + "a" * 64, + "diffs_sha256": "sha256:" + "b" * 64, + "adapter_sha256": "sha256:" + "c" * 64, + } + + _mutate_campaign_manifest(campaign_root, add_replay) + _verdicts, _stems, config = report.load_campaign("reviewbot_2026-09", 1, 2) + assert config["archived_replay"]["mode"] == "frozen_open_pr_v1" + + def corrupt(manifest): + manifest["config"]["archived_replay"]["bases_sha256"] = "bad" + + _mutate_campaign_manifest(campaign_root, corrupt) + with pytest.raises(report.CampaignError, match="archived_replay.bases_sha256"): + report.load_campaign("reviewbot_2026-09", 1, 2) + + @pytest.mark.parametrize( ("mutator", "message"), [