fix(evals): score only exact-run transcripts and preserve agent failures

.github/workflows/hermetic-evals.yml

@@ -0,0 +1,39 @@
+name: Hermetic evaluation tests
+
+on:
+  pull_request:
+    paths:
+      - "evals/hermetic/**"
+      - ".github/workflows/hermetic-evals.yml"
+  push:
+    branches: [master]
+    paths:
+      - "evals/hermetic/**"
+      - ".github/workflows/hermetic-evals.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: hermetic-evals-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Hermetic evaluation regressions
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: ./.github/actions/setup-bazel
+        with:
+          cache-key: hermetic-eval-tests
+      - name: Check runner syntax
+        run: bash -n evals/hermetic/run.sh
+      - name: Run scorer and shell-runner regressions
+        run: >-
+          bazel test
+          //evals/hermetic:score_test
+          //evals/hermetic:run_test
+          //evals/hermetic:scorecard_test
+          --test_output=errors

evals/hermetic/BUILD.bazel

@@ -8,6 +8,19 @@ py_test(
     legacy_create_init = 0,
 )
 
+py_test(
+    name = "run_test",
+    srcs = ["scorecard.py", "test_run.py"],
+    imports = ["."],
+    data = ["run.sh", "score.py"],
+    main = "test_run.py",
+    legacy_create_init = 0,
+    target_compatible_with = select({
+        "@platforms//os:windows": ["@platforms//:incompatible"],
+        "//conditions:default": [],
+    }),
+)
+
 py_test(
     name = "scorecard_test",
     srcs = ["scorecard.py", "test_scorecard.py"],

evals/hermetic/README.md

@@ -90,7 +90,7 @@ warns loudly if it does not.
   env.sh                  sourceable export block (for reuse and manual debugging)
 ```
 
-`smoke` removes the env it created on exit. A `setup` env lives until
+A successful `smoke` removes the env it created. Failed evaluations retain the isolated environment and logs. A `setup` env lives until
 `teardown --env-dir`, which refuses to delete anything that is not an
 `eval-env-*` dir under `$TMPDIR`.
 
@@ -112,7 +112,7 @@ criterion).
 Optional per-scenario fields:
 
 - **`setup_cmd`**, shell command run in `project_dir` before the agent session
-  (restore fixture state between reps).
+  (restore fixture state between reps). A nonzero setup exit records an error and skips the agent and verifier.
 - **`verify_cmd`**, shell command run in `project_dir` after the session with
   the staged dev binary first on `PATH`; non-zero exit fails the scenario.
 - **`attempt_tool`**, tool-name fragment; captured commands containing
@@ -125,8 +125,7 @@ full corpus run; `run --reps N` re-stages fixtures automatically before each
 rep after the first.
 
 `run` executes each `prompt` through the selected `--agent` inside
-`project_dir` (falling back to the indexed default project if the dir is
-missing). Claude mode recovers the `session_id` and reads the isolated
+`project_dir` (using the indexed default only when the scenario omits a project path). A missing explicit project records an error instead of borrowing the default. Claude mode recovers the `session_id` and reads the isolated
 transcript. Codex mode scores the captured `codex exec --json` stream.
 `score.py` classifies MCP tools and CLI commands separately.
 
@@ -233,3 +232,30 @@ Store the baseline the same way the post-merge re-eval protocol above prescribes
 record the per-bucket and overall adoption % as a durable fact via
 `tracedecay_fact_store_add`, so a later rerun makes any regression, or the intended
 improvement, visible against a fixed number.
+
+## Exact-run scoring and process failures
+
+The runner scores only the session belonging to the current invocation. Claude
+evidence requires the reported session ID, one exact transcript, and matching
+recorded project and session metadata; an older or foreign transcript cannot
+serve as fallback evidence. Missing, ambiguous, unreadable, or malformed
+transcripts produce an explicit error row. Codex uses this invocation's captured
+JSONL and recognizes its real MCP tool field.
+
+Each result records the actual `setup_status` and `agent_status`, or null when
+the stage was not run. A nonzero setup exit skips the agent and verifier; a
+nonzero agent exit cannot pass even if it prints a success-looking transcript.
+The `status` field distinguishes `passed`, `failed`, and `error`; `error`
+identifies the failed stage and cause. Results and a Markdown summary are
+written before the batch exits nonzero if any scenario failed or none ran.
+Artifact names include the repetition and scenario ordinal, so duplicate
+scenario IDs do not overwrite another run's evidence.
+
+Run offline behavior tests without model tokens:
+
+```sh
+bazel test //evals/hermetic:score_test //evals/hermetic:run_test \
+  //evals/hermetic:scorecard_test --test_output=errors
+```
+
+Result rows also carry `scenario_index`, the 1-based nonblank corpus entry index, stable across repetitions. The scorecard pairs indexed rows by `(scenario_index, rep)` and validates their display ID/category against the corpus. Older unindexed reports retain the unique-ID rules; mixed identity schemes are rejected. Empty or whitespace-only prompts produce a `scenario/prompt_invalid` error row without running setup, an agent, or verification.

evals/hermetic/run.sh

@@ -243,41 +243,51 @@ install_plugin() {
 
 run_agent_turn() {
   local agent="$1" model="$2" prompt="$3" cwd="$4" env_dir="$5" id="$6"
-  local out
+  local status=0
   case "${agent}" in
     claude)
-      # `</dev/null`: the agent must never inherit the caller's stdin, the
-      # corpus while-read loop feeds from it, and an agent that slurps stdin
-      # (codex exec does) would silently eat every remaining scenario line.
-      out="$(cd "${cwd}" && claude -p "${prompt}" \
+      # Do not inherit the corpus reader's stdin. Preserve the real exit code.
+      (cd "${cwd}" && claude -p "${prompt}" \
           --model "${model:-sonnet}" \
           --output-format json \
-          --dangerously-skip-permissions </dev/null 2>"${env_dir}/results/${id}.stderr")" || \
-        log "scenario ${id}: claude -p exited non-zero (see ${id}.stderr)"
-      printf '%s' "${out}" >"${env_dir}/results/${id}.claude.json"
+          --dangerously-skip-permissions) </dev/null \
+          >"${env_dir}/results/${id}.claude.json" 2>"${env_dir}/results/${id}.stderr" || status=$?
       ;;
     codex)
       local -a cmd=(codex exec --json --cd "${cwd}" --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust -c "model_reasoning_effort=\"${CODEX_REASONING_EFFORT:-medium}\"")
       if [[ -n "${model}" ]]; then
         cmd+=(--model "${model}")
       fi
       cmd+=("${prompt}")
-      if ! (cd "${cwd}" && "${cmd[@]}" </dev/null >"${env_dir}/results/${id}.codex.jsonl" 2>"${env_dir}/results/${id}.stderr"); then
-        log "scenario ${id}: codex exec exited non-zero (see ${id}.stderr)"
-      fi
+      (cd "${cwd}" && "${cmd[@]}") </dev/null \
+        >"${env_dir}/results/${id}.codex.jsonl" 2>"${env_dir}/results/${id}.stderr" || status=$?
       ;;
     *)
       die "unsupported agent: ${agent}"
       ;;
   esac
+  if [[ "${status}" -ne 0 ]]; then
+    log "scenario ${id}: ${agent} exited ${status} (see ${id}.stderr)"
+  fi
+  return "${status}"
 }
 
 score_agent_turn() {
   local agent="$1" line="$2" cwd="$3" env_dir="$4" id="$5" verify_status="${6:-}" rep="${7:-1}"
+  local setup_status="${8:-}" agent_status="${9:-}" scenario_index="${10:-}"
   local -a extra=(--rep "${rep}")
+  if [[ -n "${scenario_index}" ]]; then
+    extra+=(--scenario-index "${scenario_index}")
+  fi
   if [[ -n "${verify_status}" ]]; then
     extra+=(--verify-status "${verify_status}")
   fi
+  if [[ -n "${setup_status}" ]]; then
+    extra+=(--setup-status "${setup_status}")
+  fi
+  if [[ -n "${agent_status}" ]]; then
+    extra+=(--agent-status "${agent_status}")
+  fi
   case "${agent}" in
     claude)
       python3 "${SCRIPT_DIR}/score.py" \
@@ -436,49 +446,64 @@ run_corpus() {
       log "rep ${rep}/${reps}: resetting staged fixtures"
       stage_fixtures "${env_dir}" "${HERMETIC_TRACEDECAY_BIN}"
     fi
+    local scenario_index=0
     while IFS= read -r line; do
     [[ -z "${line}" ]] && continue
     total=$((total + 1))
+    scenario_index=$((scenario_index + 1))
 
-    local id project prompt setup_cmd verify_cmd
+    local id project prompt setup_cmd verify_cmd prompt_valid
     id="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("id",""))')"
     project="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("project_dir",""))')"
     prompt="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("prompt",""))')"
     setup_cmd="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("setup_cmd",""))')"
     verify_cmd="$(printf '%s' "${line}" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("verify_cmd",""))')"
 
-    [[ -n "${prompt}" ]] || { log "scenario ${id}: empty prompt, skipping"; continue; }
+    prompt_valid="$(printf '%s' "${line}" | python3 -c 'import json,sys; p=json.load(sys.stdin).get("prompt"); print(int(isinstance(p,str) and bool(p.strip())))')"
+    if [[ "${prompt_valid}" != "1" ]]; then
+      log "scenario ${id}: invalid prompt; recording an error without execution"
+    fi
     local run_cwd
-    run_cwd="$(resolve_project_dir "${env_dir}" "${project}")"
-    [[ -d "${run_cwd}" ]] || run_cwd="${default_project}"
-
-    local run_id="${id}"
-    [[ "${reps}" -gt 1 ]] && run_id="${id}.r${rep}"
+    if [[ -z "${project}" ]]; then
+      run_cwd="${default_project}"
+    else
+      run_cwd="$(resolve_project_dir "${env_dir}" "${project}")"
+    fi
+    if [[ ! -d "${run_cwd}" ]]; then
+      log "scenario ${id}: project directory does not exist: ${run_cwd}; skipping execution"
+    fi
 
-    if [[ -n "${setup_cmd}" ]]; then
+    # The ordinal avoids a duplicate scenario ID overwriting the earlier run.
+    local run_id="${total}.r${rep}"
+    local setup_status="" agent_status="" verify_status=""
+    if [[ "${prompt_valid}" == "1" && -d "${run_cwd}" && -n "${setup_cmd}" ]]; then
       log "scenario ${id}: setup (${setup_cmd})"
-      (cd "${run_cwd}" && bash -c "${setup_cmd}") \
-        || log "scenario ${id}: WARNING setup_cmd failed"
+      setup_status=0
+      (cd "${run_cwd}" && bash -c "${setup_cmd}") </dev/null \
+        >"${env_dir}/results/${run_id}.setup.log" 2>&1 || setup_status=$?
+      if [[ "${setup_status}" -ne 0 ]]; then
+        log "scenario ${id}: setup exited ${setup_status}; skipping agent"
+      fi
     fi
 
-    log "scenario ${id} rep ${rep}: running (agent=${label}, cwd=${run_cwd})"
-    run_agent_turn "${agent}" "${model}" "${prompt}" "${run_cwd}" "${env_dir}" "${run_id}"
+    if [[ "${prompt_valid}" == "1" && -d "${run_cwd}" && "${setup_status:-0}" -eq 0 ]]; then
+      log "scenario ${id} rep ${rep}: running (agent=${label}, cwd=${run_cwd})"
+      agent_status=0
+      run_agent_turn "${agent}" "${model}" "${prompt}" "${run_cwd}" "${env_dir}" "${run_id}" || agent_status=$?
+    fi
 
-    # Post-session effect check, with the staged dev binary first on PATH.
-    local verify_status=""
-    if [[ -n "${verify_cmd}" ]]; then
-      if (cd "${run_cwd}" && PATH="${env_dir}/bin:${PATH}" bash -c "${verify_cmd}" \
-            >"${env_dir}/results/${run_id}.verify.log" 2>&1); then
+    if [[ "${agent_status}" == "0" && -n "${verify_cmd}" ]]; then
+      if (cd "${run_cwd}" && PATH="${env_dir}/bin:${PATH}" bash -c "${verify_cmd}") </dev/null \
+            >"${env_dir}/results/${run_id}.verify.log" 2>&1; then
         verify_status=0
       else
-        verify_status=1
+        verify_status=$?
       fi
       log "scenario ${id}: verify_cmd exit ${verify_status}"
     fi
 
-    # Score: inspect the isolated transcript/output and count tools/commands.
     local scored
-    scored="$(score_agent_turn "${agent}" "${line}" "${run_cwd}" "${env_dir}" "${run_id}" "${verify_status}" "${rep}")"
+    scored="$(score_agent_turn "${agent}" "${line}" "${run_cwd}" "${env_dir}" "${run_id}" "${verify_status}" "${rep}" "${setup_status}" "${agent_status}" "${scenario_index}")"
     printf '%s\n' "${scored}" >>"${results}"
 
     if printf '%s' "${scored}" | python3 -c 'import sys,json;sys.exit(0 if json.load(sys.stdin).get("pass") else 1)'; then
@@ -498,8 +523,8 @@ run_corpus() {
     printf -- '- Agent: `%s`\n' "${label}"
     printf -- '- Dev binary: `%s`\n' "${HERMETIC_TRACEDECAY_BIN}"
     printf -- '- Passed: **%s / %s**\n\n' "${passed}" "${total}"
-    printf '| id | rep | pass | tracedecay tools | CLI commands | attempts | verify | native tools | session |\n'
-    printf '| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n'
+    printf '| id | rep | pass | error | tracedecay tools | CLI commands | attempts | verify | native tools | session |\n'
+    printf '| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n'
     python3 - "${results}" <<'PY'
 import json, sys
 with open(sys.argv[1]) as fh:
@@ -510,10 +535,11 @@ with open(sys.argv[1]) as fh:
         r = json.loads(ln)
         vp = r.get("verify_pass")
         verify = "-" if vp is None else ("yes" if vp else "no")
-        print("| {id} | {rep} | {ok} | {td} | {cli} | {attempts} | {verify} | {nat} | {sid} |".format(
+        print("| {id} | {rep} | {ok} | {error} | {td} | {cli} | {attempts} | {verify} | {nat} | {sid} |".format(
             id=r.get("id",""),
             rep=r.get("rep", 1),
             ok="yes" if r.get("pass") else "no",
+            error=(r.get("error") or {}).get("code", ""),
             td=r.get("tracedecay_tool_uses",0),
             cli=r.get("cli_command_uses",0),
             attempts=r.get("tool_cmd_attempts", ""),
@@ -529,6 +555,7 @@ PY
   log "SCORE: ${passed}/${total} passed"
   # Echo summary to stdout for the caller.
   cat "${summary}"
+  [[ "${total}" -gt 0 && "${passed}" -eq "${total}" ]]
 }
 
 # --------------------------------------------------------------------------