From b553ad0e9b410faafdd662e81bc81af22d4a4f32 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:05:00 -0700 Subject: [PATCH] fix(evals): enforce exact-run evidence and failure reporting Bind evidence to the invocation, session, project and corpus index. Preserve setup and agent exits and reject missing or invalid evidence. Include scorer, shell-runner and scorecard regression coverage. Run focused read-only harness checks on pull requests, including drafts. Fixes #4204 --- .github/workflows/hermetic-evals.yml | 39 ++++ evals/hermetic/BUILD.bazel | 13 ++ evals/hermetic/README.md | 34 +++- evals/hermetic/run.sh | 95 ++++++---- evals/hermetic/score.py | 259 +++++++++++++++++---------- evals/hermetic/scorecard.py | 15 +- evals/hermetic/test_run.py | 149 +++++++++++++++ evals/hermetic/test_score.py | 130 +++++++++++++- evals/hermetic/test_scorecard.py | 30 ++++ 9 files changed, 625 insertions(+), 139 deletions(-) create mode 100644 .github/workflows/hermetic-evals.yml diff --git a/.github/workflows/hermetic-evals.yml b/.github/workflows/hermetic-evals.yml new file mode 100644 index 0000000000..cb241e60a7 --- /dev/null +++ b/.github/workflows/hermetic-evals.yml @@ -0,0 +1,39 @@ +name: Hermetic evaluation tests + +on: + pull_request: + paths: + - "evals/hermetic/**" + - ".github/workflows/hermetic-evals.yml" + push: + branches: [master] + paths: + - "evals/hermetic/**" + - ".github/workflows/hermetic-evals.yml" + +permissions: + contents: read + +concurrency: + group: hermetic-evals-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Hermetic evaluation regressions + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/setup-bazel + with: + cache-key: hermetic-eval-tests + - name: Check runner syntax + run: bash -n evals/hermetic/run.sh + - name: Run scorer and shell-runner regressions + run: >- + bazel test + //evals/hermetic:score_test + //evals/hermetic:run_test + //evals/hermetic:scorecard_test + --test_output=errors diff --git a/evals/hermetic/BUILD.bazel b/evals/hermetic/BUILD.bazel index 33532a3f3a..b16f0187d2 100644 --- a/evals/hermetic/BUILD.bazel +++ b/evals/hermetic/BUILD.bazel @@ -8,6 +8,19 @@ py_test( legacy_create_init = 0, ) +py_test( + name = "run_test", + srcs = ["scorecard.py", "test_run.py"], + imports = ["."], + data = ["run.sh", "score.py"], + main = "test_run.py", + legacy_create_init = 0, + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), +) + py_test( name = "scorecard_test", srcs = ["scorecard.py", "test_scorecard.py"], diff --git a/evals/hermetic/README.md b/evals/hermetic/README.md index c15b46d807..e8bb5a58d0 100644 --- a/evals/hermetic/README.md +++ b/evals/hermetic/README.md @@ -90,7 +90,7 @@ warns loudly if it does not. env.sh sourceable export block (for reuse and manual debugging) ``` -`smoke` removes the env it created on exit. A `setup` env lives until +A successful `smoke` removes the env it created. Failed evaluations retain the isolated environment and logs. A `setup` env lives until `teardown --env-dir`, which refuses to delete anything that is not an `eval-env-*` dir under `$TMPDIR`. @@ -112,7 +112,7 @@ criterion). Optional per-scenario fields: - **`setup_cmd`**, shell command run in `project_dir` before the agent session - (restore fixture state between reps). + (restore fixture state between reps). A nonzero setup exit records an error and skips the agent and verifier. - **`verify_cmd`**, shell command run in `project_dir` after the session with the staged dev binary first on `PATH`; non-zero exit fails the scenario. - **`attempt_tool`**, tool-name fragment; captured commands containing @@ -125,8 +125,7 @@ full corpus run; `run --reps N` re-stages fixtures automatically before each rep after the first. `run` executes each `prompt` through the selected `--agent` inside -`project_dir` (falling back to the indexed default project if the dir is -missing). Claude mode recovers the `session_id` and reads the isolated +`project_dir` (using the indexed default only when the scenario omits a project path). A missing explicit project records an error instead of borrowing the default. Claude mode recovers the `session_id` and reads the isolated transcript. Codex mode scores the captured `codex exec --json` stream. `score.py` classifies MCP tools and CLI commands separately. @@ -233,3 +232,30 @@ Store the baseline the same way the post-merge re-eval protocol above prescribes record the per-bucket and overall adoption % as a durable fact via `tracedecay_fact_store_add`, so a later rerun makes any regression, or the intended improvement, visible against a fixed number. + +## Exact-run scoring and process failures + +The runner scores only the session belonging to the current invocation. Claude +evidence requires the reported session ID, one exact transcript, and matching +recorded project and session metadata; an older or foreign transcript cannot +serve as fallback evidence. Missing, ambiguous, unreadable, or malformed +transcripts produce an explicit error row. Codex uses this invocation's captured +JSONL and recognizes its real MCP tool field. + +Each result records the actual `setup_status` and `agent_status`, or null when +the stage was not run. A nonzero setup exit skips the agent and verifier; a +nonzero agent exit cannot pass even if it prints a success-looking transcript. +The `status` field distinguishes `passed`, `failed`, and `error`; `error` +identifies the failed stage and cause. Results and a Markdown summary are +written before the batch exits nonzero if any scenario failed or none ran. +Artifact names include the repetition and scenario ordinal, so duplicate +scenario IDs do not overwrite another run's evidence. + +Run offline behavior tests without model tokens: + +```sh +bazel test //evals/hermetic:score_test //evals/hermetic:run_test \ + //evals/hermetic:scorecard_test --test_output=errors +``` + +Result rows also carry `scenario_index`, the 1-based nonblank corpus entry index, stable across repetitions. The scorecard pairs indexed rows by `(scenario_index, rep)` and validates their display ID/category against the corpus. Older unindexed reports retain the unique-ID rules; mixed identity schemes are rejected. Empty or whitespace-only prompts produce a `scenario/prompt_invalid` error row without running setup, an agent, or verification. diff --git a/evals/hermetic/run.sh b/evals/hermetic/run.sh index c80cdd3256..fe41caf708 100755 --- a/evals/hermetic/run.sh +++ b/evals/hermetic/run.sh @@ -243,18 +243,15 @@ install_plugin() { run_agent_turn() { local agent="$1" model="$2" prompt="$3" cwd="$4" env_dir="$5" id="$6" - local out + local status=0 case "${agent}" in claude) - # `"${env_dir}/results/${id}.stderr")" || \ - log "scenario ${id}: claude -p exited non-zero (see ${id}.stderr)" - printf '%s' "${out}" >"${env_dir}/results/${id}.claude.json" + --dangerously-skip-permissions) "${env_dir}/results/${id}.claude.json" 2>"${env_dir}/results/${id}.stderr" || status=$? ;; codex) local -a cmd=(codex exec --json --cd "${cwd}" --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust -c "model_reasoning_effort=\"${CODEX_REASONING_EFFORT:-medium}\"") @@ -262,22 +259,35 @@ run_agent_turn() { cmd+=(--model "${model}") fi cmd+=("${prompt}") - if ! (cd "${cwd}" && "${cmd[@]}" "${env_dir}/results/${id}.codex.jsonl" 2>"${env_dir}/results/${id}.stderr"); then - log "scenario ${id}: codex exec exited non-zero (see ${id}.stderr)" - fi + (cd "${cwd}" && "${cmd[@]}") "${env_dir}/results/${id}.codex.jsonl" 2>"${env_dir}/results/${id}.stderr" || status=$? ;; *) die "unsupported agent: ${agent}" ;; esac + if [[ "${status}" -ne 0 ]]; then + log "scenario ${id}: ${agent} exited ${status} (see ${id}.stderr)" + fi + return "${status}" } score_agent_turn() { local agent="$1" line="$2" cwd="$3" env_dir="$4" id="$5" verify_status="${6:-}" rep="${7:-1}" + local setup_status="${8:-}" agent_status="${9:-}" scenario_index="${10:-}" local -a extra=(--rep "${rep}") + if [[ -n "${scenario_index}" ]]; then + extra+=(--scenario-index "${scenario_index}") + fi if [[ -n "${verify_status}" ]]; then extra+=(--verify-status "${verify_status}") fi + if [[ -n "${setup_status}" ]]; then + extra+=(--setup-status "${setup_status}") + fi + if [[ -n "${agent_status}" ]]; then + extra+=(--agent-status "${agent_status}") + fi case "${agent}" in claude) python3 "${SCRIPT_DIR}/score.py" \ @@ -436,49 +446,64 @@ run_corpus() { log "rep ${rep}/${reps}: resetting staged fixtures" stage_fixtures "${env_dir}" "${HERMETIC_TRACEDECAY_BIN}" fi + local scenario_index=0 while IFS= read -r line; do [[ -z "${line}" ]] && continue total=$((total + 1)) + scenario_index=$((scenario_index + 1)) - local id project prompt setup_cmd verify_cmd + local id project prompt setup_cmd verify_cmd prompt_valid id="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("id",""))')" project="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("project_dir",""))')" prompt="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("prompt",""))')" setup_cmd="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("setup_cmd",""))')" verify_cmd="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("verify_cmd",""))')" - [[ -n "${prompt}" ]] || { log "scenario ${id}: empty prompt, skipping"; continue; } + prompt_valid="$(printf '%s' "${line}" | python3 -c 'import json,sys; p=json.load(sys.stdin).get("prompt"); print(int(isinstance(p,str) and bool(p.strip())))')" + if [[ "${prompt_valid}" != "1" ]]; then + log "scenario ${id}: invalid prompt; recording an error without execution" + fi local run_cwd - run_cwd="$(resolve_project_dir "${env_dir}" "${project}")" - [[ -d "${run_cwd}" ]] || run_cwd="${default_project}" - - local run_id="${id}" - [[ "${reps}" -gt 1 ]] && run_id="${id}.r${rep}" + if [[ -z "${project}" ]]; then + run_cwd="${default_project}" + else + run_cwd="$(resolve_project_dir "${env_dir}" "${project}")" + fi + if [[ ! -d "${run_cwd}" ]]; then + log "scenario ${id}: project directory does not exist: ${run_cwd}; skipping execution" + fi - if [[ -n "${setup_cmd}" ]]; then + # The ordinal avoids a duplicate scenario ID overwriting the earlier run. + local run_id="${total}.r${rep}" + local setup_status="" agent_status="" verify_status="" + if [[ "${prompt_valid}" == "1" && -d "${run_cwd}" && -n "${setup_cmd}" ]]; then log "scenario ${id}: setup (${setup_cmd})" - (cd "${run_cwd}" && bash -c "${setup_cmd}") \ - || log "scenario ${id}: WARNING setup_cmd failed" + setup_status=0 + (cd "${run_cwd}" && bash -c "${setup_cmd}") "${env_dir}/results/${run_id}.setup.log" 2>&1 || setup_status=$? + if [[ "${setup_status}" -ne 0 ]]; then + log "scenario ${id}: setup exited ${setup_status}; skipping agent" + fi fi - log "scenario ${id} rep ${rep}: running (agent=${label}, cwd=${run_cwd})" - run_agent_turn "${agent}" "${model}" "${prompt}" "${run_cwd}" "${env_dir}" "${run_id}" + if [[ "${prompt_valid}" == "1" && -d "${run_cwd}" && "${setup_status:-0}" -eq 0 ]]; then + log "scenario ${id} rep ${rep}: running (agent=${label}, cwd=${run_cwd})" + agent_status=0 + run_agent_turn "${agent}" "${model}" "${prompt}" "${run_cwd}" "${env_dir}" "${run_id}" || agent_status=$? + fi - # Post-session effect check, with the staged dev binary first on PATH. - local verify_status="" - if [[ -n "${verify_cmd}" ]]; then - if (cd "${run_cwd}" && PATH="${env_dir}/bin:${PATH}" bash -c "${verify_cmd}" \ - >"${env_dir}/results/${run_id}.verify.log" 2>&1); then + if [[ "${agent_status}" == "0" && -n "${verify_cmd}" ]]; then + if (cd "${run_cwd}" && PATH="${env_dir}/bin:${PATH}" bash -c "${verify_cmd}") "${env_dir}/results/${run_id}.verify.log" 2>&1; then verify_status=0 else - verify_status=1 + verify_status=$? fi log "scenario ${id}: verify_cmd exit ${verify_status}" fi - # Score: inspect the isolated transcript/output and count tools/commands. local scored - scored="$(score_agent_turn "${agent}" "${line}" "${run_cwd}" "${env_dir}" "${run_id}" "${verify_status}" "${rep}")" + scored="$(score_agent_turn "${agent}" "${line}" "${run_cwd}" "${env_dir}" "${run_id}" "${verify_status}" "${rep}" "${setup_status}" "${agent_status}" "${scenario_index}")" printf '%s\n' "${scored}" >>"${results}" if printf '%s' "${scored}" | python3 -c 'import sys,json;sys.exit(0 if json.load(sys.stdin).get("pass") else 1)'; then @@ -498,8 +523,8 @@ run_corpus() { printf -- '- Agent: `%s`\n' "${label}" printf -- '- Dev binary: `%s`\n' "${HERMETIC_TRACEDECAY_BIN}" printf -- '- Passed: **%s / %s**\n\n' "${passed}" "${total}" - printf '| id | rep | pass | tracedecay tools | CLI commands | attempts | verify | native tools | session |\n' - printf '| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n' + printf '| id | rep | pass | error | tracedecay tools | CLI commands | attempts | verify | native tools | session |\n' + printf '| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n' python3 - "${results}" <<'PY' import json, sys with open(sys.argv[1]) as fh: @@ -510,10 +535,11 @@ with open(sys.argv[1]) as fh: r = json.loads(ln) vp = r.get("verify_pass") verify = "-" if vp is None else ("yes" if vp else "no") - print("| {id} | {rep} | {ok} | {td} | {cli} | {attempts} | {verify} | {nat} | {sid} |".format( + print("| {id} | {rep} | {ok} | {error} | {td} | {cli} | {attempts} | {verify} | {nat} | {sid} |".format( id=r.get("id",""), rep=r.get("rep", 1), ok="yes" if r.get("pass") else "no", + error=(r.get("error") or {}).get("code", ""), td=r.get("tracedecay_tool_uses",0), cli=r.get("cli_command_uses",0), attempts=r.get("tool_cmd_attempts", ""), @@ -529,6 +555,7 @@ PY log "SCORE: ${passed}/${total} passed" # Echo summary to stdout for the caller. cat "${summary}" + [[ "${total}" -gt 0 && "${passed}" -eq "${total}" ]] } # -------------------------------------------------------------------------- diff --git a/evals/hermetic/score.py b/evals/hermetic/score.py index b710f1f8e3..aa14b6f55b 100755 --- a/evals/hermetic/score.py +++ b/evals/hermetic/score.py @@ -40,57 +40,75 @@ def load_scenario(raw: str) -> dict: return json.loads(raw) -def session_id_from_claude_json(path: Path) -> str | None: - """Recover the session id from the `claude -p --output-format json` result.""" - if not path.exists(): - return None +class EvidenceError(ValueError): + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + +def read_jsonl(path: Path) -> list[dict]: + try: + lines = path.read_text().splitlines() + except (OSError, UnicodeError) as exc: + raise EvidenceError("transcript_unreadable", f"Cannot read {path}: {exc}") from exc + events = [] + for number, line in enumerate(lines, 1): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError as exc: + raise EvidenceError("transcript_invalid", f"Invalid JSON in {path}:{number}: {exc}") from exc + if not isinstance(event, dict): + raise EvidenceError("transcript_invalid", f"Expected an object in {path}:{number}") + events.append(event) + if not events: + raise EvidenceError("transcript_empty", f"No transcript events in {path}") + return events + + +def session_id_from_claude_json(path: Path) -> str: + """Recover only the session created by this Claude invocation.""" try: data = json.loads(path.read_text()) - except (json.JSONDecodeError, OSError): - return None + except (json.JSONDecodeError, OSError, UnicodeError) as exc: + raise EvidenceError("agent_output_invalid", f"Cannot read Claude result {path}: {exc}") from exc if isinstance(data, dict): for key in ("session_id", "sessionId", "session"): val = data.get(key) if isinstance(val, str) and val: return val - return None + raise EvidenceError("session_missing", f"Claude result {path} has no session id") -def project_slug(cwd: str) -> str: - """Claude Code stores transcripts under projects/ where slug is the - absolute cwd with path separators replaced by dashes.""" - return cwd.replace("/", "-") - - -def find_transcript(config_dir: Path, cwd: str, session_id: str | None) -> Path | None: - """Locate the JSONL transcript for this session inside the isolated config.""" +def find_transcript(config_dir: Path, cwd: str, session_id: str) -> Path: + """Match the exact session and its recorded project, not a best-effort slug.""" + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", session_id): + raise EvidenceError("session_invalid", f"Invalid session id: {session_id!r}") projects = config_dir / "projects" - candidates: list[Path] = [] - - if session_id: - # Fast path: /projects//.jsonl - slug_dir = projects / project_slug(cwd) - direct = slug_dir / f"{session_id}.jsonl" - if direct.exists(): - return direct - candidates.extend(projects.rglob(f"{session_id}.jsonl")) - if candidates: - return candidates[0] - - # Fallback: newest transcript under the matching project slug. - slug_dir = projects / project_slug(cwd) - if slug_dir.is_dir(): - jsonls = sorted( - slug_dir.glob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True - ) - if jsonls: - return jsonls[0] - - # Last resort: newest transcript anywhere in the isolated config. - all_jsonls = sorted( - projects.rglob("*.jsonl"), key=lambda p: p.stat().st_mtime, reverse=True - ) if projects.is_dir() else [] - return all_jsonls[0] if all_jsonls else None + candidates = list(projects.glob(f"*/{session_id}.jsonl")) + if not candidates: + raise EvidenceError("transcript_missing", f"No transcript for session {session_id}") + if len(candidates) != 1: + raise EvidenceError("transcript_ambiguous", f"Multiple transcripts for session {session_id}") + transcript = candidates[0] + events = read_jsonl(transcript) + sessions = set() + working_dirs = set() + for event in events: + if "sessionId" in event: + if not isinstance(event["sessionId"], str): + raise EvidenceError("session_mismatch", f"Invalid session metadata in {transcript}") + sessions.add(event["sessionId"]) + if "cwd" in event: + if not isinstance(event["cwd"], str): + raise EvidenceError("project_mismatch", f"Invalid project metadata in {transcript}") + working_dirs.add(event["cwd"]) + if sessions != {session_id}: + raise EvidenceError("session_mismatch", f"Transcript {transcript} does not identify only session {session_id}") + if not working_dirs or any(not Path(path).is_absolute() or Path(path).resolve() != Path(cwd).resolve() for path in working_dirs): + raise EvidenceError("project_mismatch", f"Transcript {transcript} does not identify project {cwd}") + return transcript def is_tracedecay_tool(name: str) -> bool: @@ -111,19 +129,7 @@ def count_claude_tools(transcript: Path) -> tuple[list[str], list[str], list[str td: list[str] = [] native: list[str] = [] commands: list[str] = [] - try: - lines = transcript.read_text().splitlines() - except OSError: - return td, native, commands - - for ln in lines: - ln = ln.strip() - if not ln: - continue - try: - evt = json.loads(ln) - except json.JSONDecodeError: - continue + for evt in read_jsonl(transcript): # tool_use entries live in message.content blocks of assistant messages. msg = evt.get("message") if isinstance(evt, dict) else None content = msg.get("content") if isinstance(msg, dict) else None @@ -153,6 +159,12 @@ def count_claude_tools(transcript: Path) -> tuple[list[str], list[str], list[str def collect_codex_evidence(value, tools: list[str], commands: list[str]) -> None: if isinstance(value, dict): + if value.get("type") == "mcp_tool_call": + tool = value.get("tool") + if not isinstance(tool, str) or not tool: + raise EvidenceError("transcript_invalid", "Codex MCP call has no tool name") + tools.append(tool) + return for key in ("name", "tool_name"): name = value.get(key) if isinstance(name, str): @@ -178,21 +190,21 @@ def count_codex_tools(jsonl_path: Path) -> tuple[list[str], list[str], list[str] td: list[str] = [] native: list[str] = [] commands: list[str] = [] - try: - lines = jsonl_path.read_text().splitlines() - except OSError: - return td, native, commands - tools: list[str] = [] - for ln in lines: - ln = ln.strip() - if not ln: - continue - try: - event = json.loads(ln) - except json.JSONDecodeError: + seen_item_ids: set[str] = set() + for event in read_jsonl(jsonl_path): + item = event.get("item") + item_id = None + if isinstance(item, dict) and event.get("type") in {"item.started", "item.completed"}: + value = item.get("id") + if isinstance(value, str) and value: + item_id = value + if item_id is not None and item_id in seen_item_ids: continue + previous_counts = (len(tools), len(commands)) collect_codex_evidence(event, tools, commands) + if item_id is not None and previous_counts != (len(tools), len(commands)): + seen_item_ids.add(item_id) for name in tools: if is_tracedecay_tool(name): @@ -304,10 +316,77 @@ def agent_turn_completed(path: Path, agent: str) -> bool: return isinstance(result, dict) and result.get("type") == "result" and result.get("subtype") == "success" and not result.get("is_error", False) events = [json.loads(line) for line in text.splitlines() if line.strip()] return any(isinstance(event, dict) and event.get("type") == "turn.completed" for event in events) and not any(isinstance(event, dict) and event.get("type") in {"turn.failed", "error"} for event in events) - except (OSError, json.JSONDecodeError): + except (OSError, UnicodeError, json.JSONDecodeError): return False +def score_run( + scenario: dict, + agent: str, + output_path: Path, + cwd: str, + config_dir: Path | None = None, + setup_status: int | None = None, + agent_status: int | None = None, + verify_status: int | None = None, + rep: int = 1, + scenario_index: int | None = None, +) -> dict: + if scenario_index is not None and (type(scenario_index) is not int or scenario_index < 1): + raise ValueError("scenario_index must be a positive integer") + sid = None + transcript = None + td_tools: list[str] = [] + native_tools: list[str] = [] + commands: list[str] = [] + completed = False + error = None + + prompt = scenario.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + error = {"stage": "scenario", "code": "prompt_invalid", "message": "Scenario prompt must be a non-empty string"} + elif not Path(cwd).is_dir(): + error = {"stage": "project", "code": "project_missing", "message": f"Project directory does not exist: {cwd}"} + elif setup_status not in (None, 0): + error = {"stage": "setup", "code": "setup_failed", "message": f"Setup exited with status {setup_status}"} + elif agent_status is None: + error = {"stage": "agent", "code": "agent_not_run", "message": "No agent exit status was recorded"} + elif agent_status != 0: + error = {"stage": "agent", "code": "agent_failed", "message": f"Agent exited with status {agent_status}"} + else: + try: + if agent == "claude": + if config_dir is None: + raise EvidenceError("config_missing", "Claude scoring requires its isolated config directory") + sid = session_id_from_claude_json(output_path) + transcript = find_transcript(config_dir, cwd, sid) + td_tools, native_tools, commands = count_claude_tools(transcript) + else: + transcript = output_path + td_tools, native_tools, commands = count_codex_tools(transcript) + completed = agent_turn_completed(output_path, agent) + if not completed: + raise EvidenceError("agent_incomplete", f"No successful {agent} completion in {output_path}") + except EvidenceError as exc: + error = {"stage": "evidence", "code": exc.code, "message": str(exc)} + + result = evaluate_scenario( + scenario, sid, transcript, td_tools, native_tools, commands, + verify_status=verify_status, rep=rep, + agent_completed=completed and error is None, + ) + result.update({ + "project_dir": cwd, + "scenario_index": scenario_index, + "setup_status": setup_status, + "agent_status": agent_status, + "agent_output": str(output_path) if agent_status is not None else None, + "error": error, + "status": "error" if error else ("passed" if result["pass"] else "failed"), + }) + return result + + def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--agent", choices=("claude", "codex"), default="claude") @@ -316,46 +395,30 @@ def main() -> int: ap.add_argument("--codex-jsonl", help="path to codex exec --json output") ap.add_argument("--config-dir", help="isolated CLAUDE_CONFIG_DIR") ap.add_argument("--cwd", required=True, help="cwd the scenario ran in") - ap.add_argument( - "--verify-status", - type=int, - choices=(0, 1), - default=None, - help="exit status from the scenario verify_cmd (0=pass, 1=fail)", - ) + ap.add_argument("--setup-status", type=int, help="setup exit status; omit when no setup command ran") + ap.add_argument("--agent-status", type=int, help="agent exit status; omit when setup prevented execution") + ap.add_argument("--verify-status", type=int, default=None, help="exit status from scenario verify_cmd") ap.add_argument("--rep", type=int, default=1, help="corpus repetition index") + ap.add_argument("--scenario-index", type=int, help="1-based nonblank corpus entry index, stable across repetitions") args = ap.parse_args() scenario = load_scenario(args.scenario) - sid = None - transcript = None - td_tools: list[str] = [] - native_tools: list[str] = [] - commands: list[str] = [] - if args.agent == "claude": if not args.claude_json or not args.config_dir: ap.error("--agent claude requires --claude-json and --config-dir") - sid = session_id_from_claude_json(Path(args.claude_json)) - transcript = find_transcript(Path(args.config_dir), args.cwd, sid) - if transcript is not None: - td_tools, native_tools, commands = count_claude_tools(transcript) - else: - if not args.codex_jsonl: - ap.error("--agent codex requires --codex-jsonl") - transcript = Path(args.codex_jsonl) - td_tools, native_tools, commands = count_codex_tools(transcript) - - result = evaluate_scenario( - scenario, - sid, - transcript, - td_tools, - native_tools, - commands, + elif not args.codex_jsonl: + ap.error("--agent codex requires --codex-jsonl") + + result = score_run( + scenario, args.agent, + Path(args.claude_json if args.agent == "claude" else args.codex_jsonl), + args.cwd, + config_dir=Path(args.config_dir) if args.config_dir else None, + setup_status=args.setup_status, + agent_status=args.agent_status, verify_status=args.verify_status, rep=args.rep, - agent_completed=agent_turn_completed(Path(args.claude_json if args.agent == "claude" else args.codex_jsonl), args.agent), + scenario_index=args.scenario_index, ) print(json.dumps(result)) return 0 diff --git a/evals/hermetic/scorecard.py b/evals/hermetic/scorecard.py index 1f5981d9e9..a051e81e3b 100644 --- a/evals/hermetic/scorecard.py +++ b/evals/hermetic/scorecard.py @@ -110,9 +110,12 @@ def aggregate(results: list[dict], corpus: list[dict] | None = None, reps: int | raise ValueError("reps must be a positive integer") if any(type(row.get("rep", 1)) is not int or row.get("rep", 1) < 1 for row in results): raise ValueError("repetition must be a positive integer") + indexed = any(row.get("scenario_index") is not None for row in results) + if indexed and any(type(row.get("scenario_index")) is not int or row["scenario_index"] < 1 for row in results): + raise ValueError("indexed results require a positive scenario_index on every row") identities = set() for row in results: - identity = (row.get("id"), row.get("rep", 1)) + identity = (row["scenario_index"] if indexed else row.get("id"), row.get("rep", 1)) if identity in identities: raise ValueError(f"duplicate scenario/repetition: {identity}") identities.add(identity) @@ -120,7 +123,15 @@ def aggregate(results: list[dict], corpus: list[dict] | None = None, reps: int | reps = observed_reps if reps is None else reps if reps < 1 or observed_reps > reps: raise ValueError("reps must cover every observed positive repetition") - if corpus is not None: + if corpus is not None and indexed: + for row in results: + index = row["scenario_index"] + if index > len(corpus): + raise ValueError("result scenario_index is outside the corpus") + scenario = corpus[index - 1] + if row.get("id") != scenario.get("id") or row.get("category", "") != scenario.get("category", ""): + raise ValueError("result scenario/category does not match its corpus index") + elif corpus is not None: categories = {scenario["id"]: scenario.get("category", "") for scenario in corpus} if len(categories) != len(corpus): raise ValueError("duplicate corpus scenario") diff --git a/evals/hermetic/test_run.py b/evals/hermetic/test_run.py index 894ef37a31..c6abab7a51 100644 --- a/evals/hermetic/test_run.py +++ b/evals/hermetic/test_run.py @@ -7,9 +7,13 @@ import shlex import shutil import subprocess +import sys +import textwrap import tempfile import unittest +from scorecard import aggregate + class HarnessProjectTests(unittest.TestCase): def setUp(self): @@ -20,6 +24,7 @@ def setUp(self): self.script = self.checkout / "evals" / "hermetic" / "run.sh" self.script.parent.mkdir(parents=True) shutil.copyfile(Path(__file__).with_name("run.sh"), self.script) + shutil.copyfile(Path(__file__).with_name("score.py"), self.script.with_name("score.py")) self.env_dir = self.root / "isolated env" self.env_dir.mkdir() self.record = self.root / "invocation.json" @@ -79,5 +84,149 @@ def test_index_failure_is_not_reported_as_success(self): self.assertEqual(json.loads(self.record.read_text())["args"], ["init", str(self.checkout)]) + def run_batch(self, scenarios, agent="claude"): + """Run a real shell corpus with controlled offline fake agent processes.""" + results_dir = self.env_dir / "results" + results_dir.mkdir() + bindir = self.env_dir / "bin" + bindir.mkdir() + config = self.env_dir / "home" / ".claude" + config.mkdir(parents=True) + (bindir / "python3").symlink_to(sys.executable) + command = bindir / agent + command.write_text(textwrap.dedent("""\ + #!/usr/bin/env python3 + import json, os, sys + from pathlib import Path + + prompt = sys.argv[sys.argv.index("-p") + 1] if "-p" in sys.argv else sys.argv[-1] + record = Path(os.environ["HARNESS_RECORD"]) + with record.open("a") as stream: + stream.write(json.dumps(prompt) + "\\n") + session = "session-" + str(len(record.read_text().splitlines())) + if Path(sys.argv[0]).name == "claude": + if prompt != "missing-transcript": + path = Path(os.environ["CLAUDE_CONFIG_DIR"]) / "projects" / "encoded" / (session + ".jsonl") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({ + "type": "assistant", "sessionId": session, "cwd": os.getcwd(), + "message": {"content": [{"type": "tool_use", "name": "tracedecay_search", "input": {}}]} + }) + "\\n") + print(json.dumps({"type": "result", "subtype": "success", "session_id": session})) + else: + print(json.dumps({"type": "item.completed", "item": {"type": "mcp_tool_call", "tool": "tracedecay_search"}})) + print(json.dumps({"type": "turn.completed"})) + if prompt == "fail-after-output": + sys.exit(17) + """)) + command.chmod(0o755) + with (self.env_dir / "env.sh").open("a") as stream: + for key, value in { + "HOME": str(self.env_dir / "home"), + "CLAUDE_CONFIG_DIR": str(config), + "CODEX_HOME": str(self.env_dir / "home" / ".codex"), + "TRACEDECAY_DATA_DIR": str(self.env_dir / "tracedecay-data"), + "TRACEDECAY_DAEMON_SOCKET": str(self.env_dir / "tracedecay-data" / "daemon.sock"), + "PATH": str(bindir) + os.pathsep + os.environ["PATH"], + }.items(): + stream.write(f"export {key}={shlex.quote(value)}\n") + corpus = self.root / "corpus.jsonl" + corpus.write_text("".join(json.dumps({ + "id": f"scenario-{index}", "expected_tools": ["tracedecay_search"], + **scenario, + }) + "\n" for index, scenario in enumerate(scenarios, 1))) + args = ["bash", str(self.script), "run", "--env-dir", str(self.env_dir), + "--corpus", str(corpus), "--agent", agent] + result = subprocess.run( + args, cwd=self.root, + env={**os.environ, "HARNESS_RECORD": str(self.record)}, + text=True, capture_output=True, check=False, + ) + self.assertTrue((results_dir / "results.jsonl").exists(), result.stderr) + self.assertTrue((results_dir / "summary.md").exists(), result.stderr) + rows = [json.loads(line) for line in (results_dir / "results.jsonl").read_text().splitlines()] + return result, rows + + def test_success_then_missing_transcript_does_not_inherit_previous_success(self): + result, rows = self.run_batch([{"prompt": "success"}, {"prompt": "missing-transcript"}]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual([row["pass"] for row in rows], [True, False]) + self.assertEqual(rows[1]["error"]["code"], "transcript_missing") + self.assertEqual(rows[1]["tracedecay_tools"], []) + self.assertIn("**1 / 2**", result.stdout) + + def test_setup_failure_records_code_and_skips_agent(self): + result, rows = self.run_batch([{ + "prompt": "must-not-run", "setup_cmd": "printf 'setup diagnostic'; exit 23", + "verify_cmd": "touch verifier-ran", + }]) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.record.exists()) + self.assertFalse((self.checkout / "verifier-ran").exists()) + self.assertEqual(rows[0]["setup_status"], 23) + self.assertIsNone(rows[0]["agent_status"]) + self.assertEqual(rows[0]["error"]["code"], "setup_failed") + + def test_failed_agent_process_cannot_pass_with_successful_output(self): + result, rows = self.run_batch([ + {"prompt": "success"}, {"prompt": "fail-after-output"}, {"prompt": "success"}, + ]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual([row["pass"] for row in rows], [True, False, True]) + self.assertEqual(rows[1]["agent_status"], 17) + self.assertEqual(rows[1]["error"]["code"], "agent_failed") + + def test_explicit_missing_project_never_falls_back(self): + missing = self.root / "missing project" + result, rows = self.run_batch([{"prompt": "must-not-run", "project_dir": str(missing)}]) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.record.exists()) + self.assertEqual(rows[0]["error"]["code"], "project_missing") + self.assertEqual(rows[0]["project_dir"], str(missing)) + + def test_codex_nonzero_exit_cannot_pass_even_after_completed_event(self): + result, rows = self.run_batch([{"prompt": "fail-after-output"}], agent="codex") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(rows[0]["agent_status"], 17) + self.assertEqual(rows[0]["error"]["code"], "agent_failed") + + + def test_duplicate_display_ids_keep_distinct_scorecard_identity(self): + result, rows = self.run_batch([ + {"id": "same-label", "category": "factstore-write", "prompt": "success"}, + {"id": "same-label", "category": "factstore-recall", "prompt": "success"}, + ]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([item["scenario_index"] for item in rows], [1, 2]) + self.assertEqual([item["id"] for item in rows], ["same-label", "same-label"]) + summary = aggregate(rows, [ + {"id": "same-label", "category": "factstore-write"}, + {"id": "same-label", "category": "factstore-recall"}, + ]) + self.assertEqual(summary["overall"]["opportunities"], 2) + self.assertEqual(summary["overall"]["triggered"], 2) + + def test_empty_prompt_retains_an_error_row_and_runs_no_downstream_effects(self): + result, rows = self.run_batch([ + {"prompt": "success"}, + {"prompt": "", "setup_cmd": "touch invalid-setup-ran", "verify_cmd": "touch invalid-verifier-ran"}, + ]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(rows), 2) + self.assertEqual([item["pass"] for item in rows], [True, False]) + self.assertEqual(rows[1]["error"]["code"], "prompt_invalid") + self.assertEqual(rows[1]["error"]["stage"], "scenario") + self.assertIsNone(rows[1]["agent_status"]) + self.assertEqual(len(self.record.read_text().splitlines()), 1) + self.assertFalse((self.checkout / "invalid-setup-ran").exists()) + self.assertFalse((self.checkout / "invalid-verifier-ran").exists()) + + def test_whitespace_prompt_is_reported_without_starting_an_agent(self): + result, rows = self.run_batch([{"prompt": " \t "}]) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.record.exists()) + self.assertEqual(rows[0]["error"]["code"], "prompt_invalid") + + if __name__ == "__main__": unittest.main() diff --git a/evals/hermetic/test_score.py b/evals/hermetic/test_score.py index 00560cc348..b9c65703ac 100644 --- a/evals/hermetic/test_score.py +++ b/evals/hermetic/test_score.py @@ -1,5 +1,8 @@ +import json +from pathlib import Path +import tempfile import unittest -from score import evaluate_scenario +from score import evaluate_scenario, score_run class NegativeExpectationTests(unittest.TestCase): def grade(self, calls, status=0): @@ -24,5 +27,130 @@ def test_native_read_ban_does_not_ban_tracedecay_read(self): def test_failed_verification_still_fails(self): self.assertFalse(self.grade([], 1)["pass"]) +class ExactRunEvidenceTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.project = self.root / "project with spaces" + self.project.mkdir() + self.config = self.root / "claude" + self.output = self.root / "result.json" + self.scenario = {"id": "exact-run", "prompt": "Search this project", "expected_tools": ["tracedecay_search"]} + self.write_result() + + def write_result(self, session="current-session"): + response = {"type": "result", "subtype": "success", "is_error": False} + if session is not None: + response["session_id"] = session + self.output.write_text(json.dumps(response)) + + def write_transcript(self, session="current-session", project=None, directory="encoded-project"): + path = self.config / "projects" / directory / f"{session}.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({ + "type": "assistant", "sessionId": session, "cwd": str(project or self.project), + "message": {"content": [{"type": "tool_use", "name": "tracedecay_search", "input": {}}]}, + }) + "\n") + return path + + def grade(self, **kwargs): + args = dict( + scenario=self.scenario, agent="claude", output_path=self.output, + cwd=str(self.project), config_dir=self.config, agent_status=0, + ) + args.update(kwargs) + return score_run(**args) + + def rejected(self, result, code): + self.assertFalse(result["pass"], result) + self.assertEqual(result["status"], "error") + self.assertEqual(result["error"]["code"], code) + + def test_exact_session_and_recorded_project_pass(self): + transcript = self.write_transcript() + result = self.grade() + self.assertTrue(result["pass"], result) + self.assertEqual(result["transcript"], str(transcript)) + self.assertEqual(result["tracedecay_tools"], ["tracedecay_search"]) + + def test_missing_session_never_borrows_old_or_other_project(self): + self.write_transcript(session="older-session") + self.write_transcript(session="foreign-session", project=self.root / "foreign", directory="other") + result = self.grade() + self.rejected(result, "transcript_missing") + self.assertEqual(result["tracedecay_tools"], []) + self.assertIsNone(result["transcript"]) + + def test_missing_session_identifier_does_not_guess_latest(self): + self.write_transcript() + self.write_result(session=None) + self.rejected(self.grade(), "session_missing") + + def test_project_mismatch_is_not_a_match(self): + self.write_transcript(project=self.root / "foreign") + self.rejected(self.grade(), "project_mismatch") + + def test_ambiguous_session_files_are_rejected(self): + self.write_transcript() + self.write_transcript(directory="another-project") + self.rejected(self.grade(), "transcript_ambiguous") + + def test_malformed_transcript_is_not_partial_success(self): + path = self.write_transcript() + with path.open("a") as stream: + stream.write("{truncated\n") + result = self.grade() + self.rejected(result, "transcript_invalid") + self.assertEqual(result["tracedecay_tools"], []) + + def test_nonzero_agent_exit_invalidates_successful_output(self): + self.write_transcript() + result = self.grade(agent_status=17) + self.rejected(result, "agent_failed") + self.assertEqual(result["agent_status"], 17) + self.assertEqual(result["tracedecay_tools"], []) + + def test_setup_failure_cannot_use_stale_agent_output(self): + self.write_transcript() + result = self.grade(setup_status=23, agent_status=None) + self.rejected(result, "setup_failed") + self.assertEqual(result["tracedecay_tools"], []) + self.assertIsNone(result["agent_output"]) + + def test_unrecorded_agent_exit_cannot_assume_success(self): + self.write_transcript() + self.rejected(self.grade(agent_status=None), "agent_not_run") + + def test_codex_uses_real_mcp_tool_field_and_rejects_corrupt_output(self): + events = [ + {"type": "item.completed", "item": {"type": "mcp_tool_call", "tool": "tracedecay_search"}}, + {"type": "turn.completed"}, + ] + self.output.write_text("\n".join(json.dumps(event) for event in events) + "\n") + result = self.grade(agent="codex") + self.assertTrue(result["pass"], result) + self.assertEqual(result["tracedecay_tools"], ["tracedecay_search"]) + with self.output.open("a") as stream: + stream.write("not json\n") + self.rejected(self.grade(agent="codex"), "transcript_invalid") + + def test_codex_failure_event_overrides_completed(self): + self.output.write_text('{"type":"turn.completed"}\n{"type":"turn.failed"}\n') + result = self.grade(agent="codex", scenario={"prompt": "Do not store this", "anti_tools": ["tracedecay_fact_store_add"]}) + self.rejected(result, "agent_incomplete") + + def test_codex_mcp_tool_is_not_counted_twice_across_events(self): + events = [ + {"type": "item.started", "item": {"id": "call1", "type": "mcp_tool_call", "tool": "tracedecay_search"}}, + {"type": "item.completed", "item": {"id": "call1", "type": "mcp_tool_call", "tool": "tracedecay_search"}}, + {"type": "turn.completed"}, + ] + self.output.write_text("\n".join(json.dumps(event) for event in events) + "\n") + result = self.grade(agent="codex") + self.assertTrue(result["pass"], result) + self.assertEqual(result["tracedecay_tool_uses"], 1) + + if __name__ == "__main__": unittest.main() diff --git a/evals/hermetic/test_scorecard.py b/evals/hermetic/test_scorecard.py index 3d2a36e242..cb2c5453ee 100644 --- a/evals/hermetic/test_scorecard.py +++ b/evals/hermetic/test_scorecard.py @@ -209,5 +209,35 @@ def test_missing_all_runs_still_counts_requested_repetitions(self): self.assertEqual(result["overall"]["opportunities"], 3) self.assertEqual(result["overall"]["triggered"], 0) +class CorpusIndexTest(unittest.TestCase): + def test_repeated_labels_and_repetitions_remain_separate(self): + corpus = [{"id": "same", "category": "factstore-write"}, {"id": "same", "category": "factstore-recall"}] + rows = [row("same", scenario["category"], True, scenario_index=index, rep=rep) + for rep in (1, 2) for index, scenario in enumerate(corpus, 1)] + summary = scorecard.aggregate(rows, corpus, reps=2) + self.assertEqual(summary["overall"]["opportunities"], 4) + self.assertEqual(summary["overall"]["triggered"], 4) + + def test_duplicate_index_cannot_double_count_an_opportunity(self): + rows = [row("same", "factstore-write", True, scenario_index=1)] * 2 + with self.assertRaisesRegex(ValueError, "duplicate scenario/repetition"): + scorecard.aggregate(rows) + + def test_index_does_not_override_the_corpus_identity(self): + corpus = [{"id": "expected", "category": "factstore-write"}] + for bad in [row("foreign", "factstore-write", True, scenario_index=1), + row("expected", "factstore-recall", True, scenario_index=1), + row("expected", "factstore-write", True, scenario_index=2)]: + with self.subTest(row=bad), self.assertRaises(ValueError): + scorecard.aggregate([bad], corpus) + + def test_mixed_or_invalid_indices_are_refused(self): + for index in (None, 0, True, -1): + rows = [row("a", "factstore-write", True, scenario_index=1), + row("b", "factstore-write", True, scenario_index=index)] + with self.subTest(index=index), self.assertRaises(ValueError): + scorecard.aggregate(rows) + + if __name__ == "__main__": unittest.main(verbosity=2)