From 0bc752744e1365782e41dd5de1e006e434f7697b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 03:05:39 +0000 Subject: [PATCH] feat: per-script smoke timings as a standing dataset (#264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner's per-entry timings existed only as `PASS — s` lines in job logs, recovered by hand-scraping every time a timing question came up. Since the smoke-runner delegation (#260-#263) all ten workspace runners are thin shims over `run_python.py`, so recording them routinely is one change here rather than ten repo sweeps. `RunReport.write()` now also emits `smoke_timings.json` into the report dir: one row per entry — `{entry, kind, status, seconds, cap_s, exit_code}` — plus the run metadata (project, directory, run_type, env_profile, python, ts) and a `legs` list. `seconds` is the runner's own `time.time()` measurement, the same number the console line prints, never re-derived. `cap_s` is `timeout_for`'s resolved cap, so a TIMEOUT row records the cap it hit. `ScriptResult` gained `cap_seconds` and `exit_code`, set by `build_util` at every execution site — durations were already recorded, caps and exit codes were not. They are deliberately absent from `to_dict()`: that dict is the per-run JSON PyAutoHeart's `script_timing`/`test_run` and `aggregate_results` read, and its shape stays byte-compatible. One timings file per report DIRECTORY, not per leg. A report dir receives several runner invocations (a gate's script and notebook legs; every directory of every workspace in the `run_all` mega-run), so each write merges its rows into the existing file keyed on the entry path: both legs survive, re-running a leg replaces its own rows, `legs` records every contributor. An unreadable file is replaced rather than merged. `aggregate_results` skips it by name — it is a sidecar, not a run report, and would otherwise enter `runs` as an empty phantom. An entry that never ran carries `"seconds": null`, never a fabricated zero: that is what `was_timed` is for, and it covers both a skipped entry and one listed but missing. With `$GITHUB_STEP_SUMMARY` set, each write appends a slowest-first table of its own entries to the job summary, so a run's timings are one click away in the Actions UI. Off CI nothing is appended, and a write failure is reported and swallowed — losing the summary must not fail a run whose scripts passed. Also fixes the notebook leg's surface statement: `run.py` never passed `env_profile` to its report, so every notebook report claimed "unknown" while running under a resolved profile. The timing dataset inherits that field, where an unknown surface makes two runs incomparable for exactly the reason `_surface` exists. PyAutoHeart's `smoke-tests.yml` uploads the report dir as the companion change, which is what makes the dataset outlive the job. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EoDPz2LevKeBaDwqFKtZrU --- autohands/aggregate_results.py | 9 +- autohands/build_util.py | 14 ++ autohands/result_collector.py | 234 ++++++++++++++++++++++ autohands/run.py | 7 + docs/internals.md | 35 ++++ tests/test_result_collector.py | 343 +++++++++++++++++++++++++++++++++ 6 files changed, 641 insertions(+), 1 deletion(-) diff --git a/autohands/aggregate_results.py b/autohands/aggregate_results.py index 19db180a..41a8a116 100644 --- a/autohands/aggregate_results.py +++ b/autohands/aggregate_results.py @@ -131,7 +131,14 @@ def _surface(runs: list, script_count: int) -> dict: def aggregate(results_dir: Path) -> dict: """Read all JSON result files and produce a consolidated report.""" - json_files = sorted(results_dir.glob("**/*.json")) + # The timing dataset lives in the same directory but is not a run report — + # it has no ``results`` key and would enter ``runs`` as an empty phantom + # run, so it is excluded by name here rather than by shape. + from result_collector import TIMINGS_FILENAME + + json_files = sorted( + p for p in results_dir.glob("**/*.json") if p.name != TIMINGS_FILENAME + ) if not json_files: print(f"No JSON result files found in {results_dir}", file=sys.stderr) return { diff --git a/autohands/build_util.py b/autohands/build_util.py index 7c6e039c..f0fb96dd 100644 --- a/autohands/build_util.py +++ b/autohands/build_util.py @@ -490,6 +490,7 @@ def _classify_notebook_run(run_target, recorded, report, env, timeout_secs): status=Status.TIMEOUT, duration_seconds=duration, error_message=message, + cap_seconds=timeout_secs, )) return "timeout" logging.exception(e) @@ -509,6 +510,8 @@ def _classify_notebook_run(run_target, recorded, report, env, timeout_secs): status=Status.PASSED, duration_seconds=duration, error_message="sys.exit(0) skip guard (ignored)", + cap_seconds=timeout_secs, + exit_code=e.returncode, )) else: print(f" PASS (skipped via sys.exit(0), {duration:.1f}s)") @@ -523,6 +526,8 @@ def _classify_notebook_run(run_target, recorded, report, env, timeout_secs): status=Status.PASSED, duration_seconds=duration, error_message="InversionException (ignored)", + cap_seconds=timeout_secs, + exit_code=e.returncode, )) return "passed" @@ -537,6 +542,8 @@ def _classify_notebook_run(run_target, recorded, report, env, timeout_secs): duration_seconds=duration, error_message=str(e), traceback=stderr, + cap_seconds=timeout_secs, + exit_code=e.returncode, )) return "failed" # stderr is captured now (see the subprocess call above), so echo it @@ -554,6 +561,8 @@ def _classify_notebook_run(run_target, recorded, report, env, timeout_secs): file=recorded, status=Status.PASSED, duration_seconds=duration, + cap_seconds=timeout_secs, + exit_code=0, )) return "passed" @@ -758,6 +767,7 @@ def execute_script(f, report=None, env=None, extra_args=None): status=Status.TIMEOUT, duration_seconds=duration, error_message=message, + cap_seconds=timeout_secs, )) return logging.exception(e) @@ -777,6 +787,8 @@ def execute_script(f, report=None, env=None, extra_args=None): duration_seconds=duration, error_message=str(e), traceback=stderr, + cap_seconds=timeout_secs, + exit_code=e.returncode, )) return logging.exception(e) @@ -790,6 +802,8 @@ def execute_script(f, report=None, env=None, extra_args=None): file=str(f), status=Status.PASSED, duration_seconds=duration, + cap_seconds=timeout_secs, + exit_code=0, )) diff --git a/autohands/result_collector.py b/autohands/result_collector.py index db693d62..86a68d5b 100644 --- a/autohands/result_collector.py +++ b/autohands/result_collector.py @@ -1,10 +1,25 @@ import dataclasses import datetime import json +import os +import sys from enum import Enum from pathlib import Path from typing import List, Optional +# The consolidated per-entry timing dataset written alongside every report. +# +# Why a second file rather than more keys on the per-run JSON: the per-run +# ``____.json`` files are read by PyAutoHeart +# (``script_timing`` globs ``*__script.json``, ``test_run`` reads the +# aggregated ``report.json``) and by ``aggregate_results``. Their shape is a +# published interface, so the timing dataset is emitted beside them instead — +# one file per report DIRECTORY, merged across the legs that write into it, so +# a run's timings are a single artifact regardless of how many runner +# invocations produced them. +TIMINGS_FILENAME = "smoke_timings.json" +TIMINGS_SCHEMA = "smoke_timings/1" + class Status(str, Enum): PASSED = "passed" @@ -13,6 +28,24 @@ class Status(str, Enum): TIMEOUT = "timeout" +def workspace_relative(path: str) -> str: + """Render a recorded result path relative to the workspace root (cwd). + + The runners record absolute paths (``find_scripts_in_folder`` and + ``files_from_list`` both build from ``Path.cwd()``). The timing dataset is + compared ACROSS runs and machines, where an absolute path is noise and, on + a GitHub runner, a different string every time. A path outside the + workspace, or an already-relative one, is returned unchanged. + """ + p = Path(path) + if not p.is_absolute(): + return str(p) + try: + return str(p.relative_to(Path.cwd())) + except ValueError: # pragma: no cover - a path outside the workspace + return str(p) + + @dataclasses.dataclass class ScriptResult: file: str @@ -21,8 +54,55 @@ class ScriptResult: error_message: Optional[str] = None traceback: Optional[str] = None skip_reason: Optional[str] = None + # The wall-clock cap that was in force for this entry, as resolved by + # ``build_util.timeout_for`` at the execution site (a profile's + # ``BUILD_SCRIPT_TIMEOUT`` override, else the ambient global). Only an + # entry that actually entered an execution carries one, so ``None`` is the + # marker for "never ran" — which is what keeps a skipped entry out of the + # timing dataset instead of being recorded as a 0-second run. + cap_seconds: Optional[float] = None + # The child's exit status. ``None`` for a timeout (the process group was + # killed, so there is no exit code the script chose) and for entries that + # never ran. + exit_code: Optional[int] = None + + @property + def was_timed(self) -> bool: + """True when this entry actually ran and its duration is a measurement. + + A SKIPPED entry never started, and a listed-but-missing entry fails + before any execution — both carry ``duration_seconds == 0.0`` purely as + the dataclass default. Recording those as "0 seconds" would put + fabricated rows in a dataset whose whole purpose is timing, so they are + emitted with a null duration instead. + """ + if self.status == Status.SKIPPED: + return False + return self.cap_seconds is not None or self.duration_seconds > 0 + + def to_timings_entry(self) -> dict: + """One row of the timing dataset. + + ``seconds`` is the runner's OWN measurement — the same + ``time.time()`` delta ``build_util`` prints on the ``PASS`` / + ``TIMEOUT`` line — never re-derived from timestamps elsewhere. + """ + path = workspace_relative(self.file) + return { + "entry": path, + "kind": "notebook" if path.endswith(".ipynb") else "script", + "status": self.status.value, + "seconds": round(self.duration_seconds, 2) if self.was_timed else None, + "cap_s": self.cap_seconds, + "exit_code": self.exit_code, + } def to_dict(self): + # ``cap_seconds`` / ``exit_code`` are deliberately NOT emitted here. + # This dict is the per-run JSON that PyAutoHeart's ``script_timing`` + # and ``test_run`` checks and ``aggregate_results`` read; it stays + # byte-compatible, and the new fields reach consumers through + # ``to_timings_entry`` instead. d = { "file": self.file, "status": self.status.value, @@ -144,6 +224,154 @@ def to_markdown(self) -> str: return "\n".join(lines) + # --- the timing dataset (one file per report dir) ------------------------- + + def _leg(self) -> dict: + """This report's identity within a shared report directory. + + A report dir can receive several runner invocations — the script leg + and the notebook leg of one smoke gate, or every directory of every + workspace in the ``run_all`` mega-run. Each is a *leg*, and the merged + timing file records them all so the dataset states what produced it. + """ + return { + "project": self.project, + "directory": self.directory, + "run_type": self.run_type, + "env_profile": self.env_profile, + "ts": self.completed_at or self.started_at, + "entries": len(self.results), + } + + def to_timings(self) -> dict: + """The timing dataset for THIS report, before merging.""" + return { + "schema": TIMINGS_SCHEMA, + "project": self.project, + "directory": self.directory, + "run_type": self.run_type, + "env_profile": self.env_profile, + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "ts": self.completed_at or self.started_at, + "entries": [r.to_timings_entry() for r in self.results], + "legs": [self._leg()], + } + + def merge_timings(self, existing: Optional[dict]) -> dict: + """Fold this report's entries into an already-written timing dataset. + + The merge key is the workspace-relative entry path: the script leg and + the notebook leg contribute disjoint paths, so both survive, while + re-running the SAME leg replaces its own rows rather than duplicating + them (a runner invoked twice into one report dir must not double-count). + + The top-level metadata describes the leg that wrote last; ``legs`` + carries every contributing leg, which is what a report dir spanning + more than one project (the ``run_all`` mega-run) needs in order to be + read back honestly. + + An unreadable or foreign file is replaced rather than merged — a + corrupt sidecar must not take the run down or silently poison the + dataset. + """ + fresh = self.to_timings() + if not isinstance(existing, dict) or existing.get("schema") != TIMINGS_SCHEMA: + return fresh + + mine = {e["entry"] for e in fresh["entries"]} + prior = [ + e + for e in existing.get("entries", []) + if isinstance(e, dict) and e.get("entry") not in mine + ] + fresh["entries"] = prior + fresh["entries"] + + def key(leg): + return (leg.get("project"), leg.get("directory"), leg.get("run_type")) + + mine_key = key(self._leg()) + prior_legs = [ + leg + for leg in existing.get("legs", []) + if isinstance(leg, dict) and key(leg) != mine_key + ] + fresh["legs"] = prior_legs + fresh["legs"] + return fresh + + def write_timings(self, output_dir: Path) -> Path: + path = output_dir / TIMINGS_FILENAME + existing = None + if path.exists(): + try: + existing = json.loads(path.read_text()) + except (json.JSONDecodeError, OSError, UnicodeDecodeError): + existing = None + with open(path, "w") as f: + json.dump(self.merge_timings(existing), f, indent=2) + return path + + def timings_markdown(self) -> str: + """A slowest-first timing table for the GitHub Actions step summary. + + Only this report's own entries: the step summary is append-only, so a + second leg adds its own table rather than restating the first's. + """ + timed = [r for r in self.results if r.was_timed] + untimed = [r for r in self.results if not r.was_timed] + timed.sort(key=lambda r: r.duration_seconds, reverse=True) + + total = round(sum(r.duration_seconds for r in timed), 1) + lines = [ + "", + f"### Smoke timings — {self.project} / {self.directory} " + f"({self.run_type}, Python " + f"{sys.version_info.major}.{sys.version_info.minor})", + "", + "| Entry | Status | Seconds | Cap |", + "|---|---|---:|---:|", + ] + for r in timed + untimed: + entry = workspace_relative(r.file) + seconds = f"{r.duration_seconds:.1f}" if r.was_timed else "—" + # The cap is only informative where it BOUND the entry: on a + # passing script it is the same number on every row and reads as + # noise, while on a timeout it is the whole story. + cap = ( + f"{r.cap_seconds:.0f}s" + if r.status == Status.TIMEOUT and r.cap_seconds is not None + else "" + ) + lines.append( + f"| `{entry}` | {r.status.value} | {seconds} | {cap} |" + ) + lines.append("") + count = len(self.results) + lines.append( + f"**{count} {'entry' if count == 1 else 'entries'}** | " + f"{len(timed)} timed | {total}s total" + ) + lines.append("") + return "\n".join(lines) + + def append_step_summary(self) -> bool: + """Append the timing table to ``$GITHUB_STEP_SUMMARY`` when in Actions. + + Returns False (and changes nothing) off CI, so a local run is + byte-identical to before. A write failure is reported and swallowed: + the summary is a convenience, and losing it must not fail a run whose + scripts all passed. + """ + target = os.environ.get("GITHUB_STEP_SUMMARY") + if not target: + return False + try: + with open(target, "a") as f: + f.write(self.timings_markdown()) + except OSError as exc: + print(f" [smoke timings] step summary not written: {exc}") + return False + return True + def write(self, output_dir: Path): self.completed_at = datetime.datetime.now().isoformat() output_dir.mkdir(parents=True, exist_ok=True) @@ -158,6 +386,12 @@ def write(self, output_dir: Path): with open(md_path, "w") as f: f.write(self.to_markdown()) + # Every report contributes to the standing timing dataset, and to the + # Actions step summary when there is one. Both legs (run_python.py and + # run.py) reach this same call, so neither needs its own emission. + self.write_timings(output_dir) + self.append_step_summary() + return json_path diff --git a/autohands/run.py b/autohands/run.py index 1fe32f2c..9f4e1704 100644 --- a/autohands/run.py +++ b/autohands/run.py @@ -123,6 +123,13 @@ project=project, directory=directory, run_type="notebook", + # Same surface statement the script leg has recorded since + # PyAutoHeart#83 §5.3. It was missing here, so every notebook + # report claimed env_profile "unknown" while running under a + # resolved profile — and the timing dataset (PyAutoHands#264) + # inherits this field, where an unknown surface makes two runs + # incomparable for exactly the reason _surface exists. + env_profile=(env_config_path.name if env_config_path else "none"), ) # Only when the policy file exists: with an explicit list it may be # absent, and there are then no skip reasons to parse. diff --git a/docs/internals.md b/docs/internals.md index 8d00abc9..bb2bcf97 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -123,6 +123,41 @@ All scripts in `autohands/` are run from within a checked-out workspace director - **`url_check`** — URL hygiene moved to PyAutoHeart (Heart owns all health checking). `autohands url_check` is now a thin shim to `pyauto-heart url_check`; the ecosystem-wide sweep runs from PyAutoHeart's central `url-check.yml` workflow (replacing the old per-repo `url_check.yml` workflows). The runnable scripts live at `PyAutoHeart/heart/checks/url_check*.{sh,py}`. - **`bump_colab_urls.sh `** — Rewrites every `colab.research.google.com/github/PyAutoLabs//blob//...` URL in cwd to use ``, where `` is one of `autofit_workspace`, `autogalaxy_workspace`, `autolens_workspace`, `HowToFit`, `HowToGalaxy`, `HowToLens`. Called by the `release_workspaces` and `bump_library_colab_urls` jobs in `release.yml` so README/docs Colab links always pin to the just-released tag. Idempotent; skips URLs not in canonical PyAutoLabs/date-tagged form. +### What `--report-dir` contains + +Every `RunReport.write()` (so `run_python.py`, `run.py` and `generate.py`, on +every leg that is given `--report-dir`) puts three things in that directory: + +- `____.json` — the per-run report. Read by + PyAutoHeart's `script_timing` (which globs `*__script.json`) and, via + `aggregate_results`, by its `test_run` check. **Its shape is a published + interface** — add fields to the timing dataset below, not to this. +- `____.md` — the human-readable form. +- `smoke_timings.json` — the standing per-entry **timing dataset**: one row per + entry, `{entry, kind, status, seconds, cap_s, exit_code}`, plus the run + metadata (`project`, `directory`, `run_type`, `env_profile`, `python`, `ts`) + and a `legs` list. `seconds` is the runner's own `time.time()` measurement — + the same number the `PASS`/`TIMEOUT` console line prints, never re-derived — + and `cap_s` is `build_util.timeout_for`'s resolved cap, so a TIMEOUT row + records the cap it hit. An entry that never ran (skipped, or listed but + missing) carries `"seconds": null` rather than a fabricated zero. + + There is **one** timings file per report directory, not one per leg: a + directory receives several runner invocations (a smoke gate's script and + notebook legs; every directory of every workspace in the `run_all` + mega-run), and each `write()` merges its rows into the existing file keyed on + the entry path. So the script leg and notebook leg both survive, re-running a + leg replaces its own rows, and the top-level metadata describes the leg that + wrote last while `legs` records every contributor. `aggregate_results` skips + the file by name — it is a sidecar, not a run report. + + When `$GITHUB_STEP_SUMMARY` is set (i.e. in Actions), each `write()` also + appends a slowest-first markdown table of its own entries to the job summary, + so a run's timings are readable without downloading an artifact. Off CI + nothing is appended. PyAutoHeart's reusable `smoke-tests.yml` uploads the + report directory as `smoke-timings-`, which is what makes the + dataset persist beyond the job. + ## Architecture ### Script-to-Notebook Conversion Pipeline diff --git a/tests/test_result_collector.py b/tests/test_result_collector.py index debfb9a3..e83b7d3c 100644 --- a/tests/test_result_collector.py +++ b/tests/test_result_collector.py @@ -137,3 +137,346 @@ def test_run_report_env_profile_defaults_to_unknown(): from result_collector import RunReport r = RunReport(project="p", directory="d", run_type="script") assert r.to_dict()["env_profile"] == "unknown" + + +# --- the timing dataset (PyAutoHands#264) ------------------------------------- +# +# Repo/project names below are fabricated fixtures, not real workspaces. + +import os + +from result_collector import TIMINGS_FILENAME, TIMINGS_SCHEMA + + +def _report(**kwargs): + defaults = dict( + project="widgets", + directory="scripts/demo", + run_type="script", + env_profile="profile_smoke.yaml", + ) + defaults.update(kwargs) + return RunReport(**defaults) + + +def test_timings_entry_carries_the_full_row(): + r = ScriptResult( + file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=12.345, + cap_seconds=300, + exit_code=0, + ) + assert r.to_timings_entry() == { + "entry": "scripts/demo/alpha.py", + "kind": "script", + "status": "passed", + "seconds": 12.35, + "cap_s": 300, + "exit_code": 0, + } + + +def test_timings_entry_kind_follows_the_suffix(): + nb = ScriptResult(file="notebooks/demo/alpha.ipynb", status=Status.PASSED, + duration_seconds=1.0, cap_seconds=60, exit_code=0) + assert nb.to_timings_entry()["kind"] == "notebook" + + +def test_timings_entry_is_workspace_relative(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + absolute = tmp_path / "scripts" / "demo" / "alpha.py" + r = ScriptResult(file=str(absolute), status=Status.PASSED, + duration_seconds=2.0, cap_seconds=300, exit_code=0) + assert r.to_timings_entry()["entry"] == "scripts/demo/alpha.py" + + +def test_timeout_entry_records_the_cap_it_hit(): + r = ScriptResult( + file="scripts/demo/slow.py", + status=Status.TIMEOUT, + duration_seconds=300.4, + cap_seconds=300, + ) + entry = r.to_timings_entry() + assert entry["status"] == "timeout" + assert entry["cap_s"] == 300 + assert entry["seconds"] == 300.4 + # A killed process group chose no exit code. + assert entry["exit_code"] is None + + +def test_skipped_entry_is_never_fabricated_as_zero_seconds(): + r = ScriptResult(file="scripts/demo/gui.py", status=Status.SKIPPED, + skip_reason="GUI script") + entry = r.to_timings_entry() + assert entry["status"] == "skipped" + assert entry["seconds"] is None + assert entry["cap_s"] is None + + +def test_listed_but_missing_entry_is_untimed(): + # Never entered an execution, so its 0.0 duration is the dataclass default, + # not a measurement. + r = ScriptResult(file="scripts/demo/gone.py", status=Status.FAILED, + error_message="Listed in the script list but not found") + assert r.to_timings_entry()["seconds"] is None + + +def test_write_emits_the_timings_file(tmp_path): + report = _report() + report.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=3.0, cap_seconds=300, + exit_code=0)) + report.write(tmp_path) + + data = json.loads((tmp_path / TIMINGS_FILENAME).read_text()) + assert data["schema"] == TIMINGS_SCHEMA + assert data["project"] == "widgets" + assert data["directory"] == "scripts/demo" + assert data["run_type"] == "script" + assert data["env_profile"] == "profile_smoke.yaml" + assert data["python"] == f"{sys.version_info.major}.{sys.version_info.minor}" + assert data["ts"] + assert [e["entry"] for e in data["entries"]] == ["scripts/demo/alpha.py"] + assert [leg["run_type"] for leg in data["legs"]] == ["script"] + + +def test_notebook_leg_does_not_clobber_the_script_leg(tmp_path): + scripts = _report() + scripts.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=3.0, cap_seconds=300, + exit_code=0)) + scripts.write(tmp_path) + + notebooks = _report(directory="notebooks/demo", run_type="notebook") + notebooks.results.append(ScriptResult(file="notebooks/demo/alpha.ipynb", + status=Status.PASSED, + duration_seconds=9.0, cap_seconds=300, + exit_code=0)) + notebooks.write(tmp_path) + + data = json.loads((tmp_path / TIMINGS_FILENAME).read_text()) + assert [e["entry"] for e in data["entries"]] == [ + "scripts/demo/alpha.py", + "notebooks/demo/alpha.ipynb", + ] + assert [e["kind"] for e in data["entries"]] == ["script", "notebook"] + assert {leg["run_type"] for leg in data["legs"]} == {"script", "notebook"} + + +def test_rerunning_one_leg_replaces_its_own_rows(tmp_path): + first = _report() + first.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=3.0, cap_seconds=300, + exit_code=0)) + first.write(tmp_path) + + second = _report() + second.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=8.0, cap_seconds=300, + exit_code=0)) + second.write(tmp_path) + + data = json.loads((tmp_path / TIMINGS_FILENAME).read_text()) + assert len(data["entries"]) == 1 + assert data["entries"][0]["seconds"] == 8.0 + assert len(data["legs"]) == 1 + + +def test_corrupt_timings_file_is_replaced_not_fatal(tmp_path): + (tmp_path).mkdir(parents=True, exist_ok=True) + (tmp_path / TIMINGS_FILENAME).write_text("{ not json") + + report = _report() + report.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=1.0, cap_seconds=300, + exit_code=0)) + report.write(tmp_path) + + data = json.loads((tmp_path / TIMINGS_FILENAME).read_text()) + assert len(data["entries"]) == 1 + + +def test_per_run_json_shape_is_untouched(tmp_path): + # The published interface PyAutoHeart's script_timing/test_run read. + report = _report() + report.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=3.0, cap_seconds=300, + exit_code=0)) + path = report.write(tmp_path) + result = json.loads(path.read_text())["results"][0] + assert set(result) == {"file", "status", "duration_seconds"} + + +def test_aggregate_ignores_the_timings_file(tmp_path): + from aggregate_results import aggregate + + report = _report() + report.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=3.0, cap_seconds=300, + exit_code=0)) + report.write(tmp_path) + + out = aggregate(tmp_path) + assert len(out["runs"]) == 1 + assert out["runs"][0]["project"] == "widgets" + + +# --- the step summary --------------------------------------------------------- + + +def test_step_summary_table_is_slowest_first(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + report = _report() + report.results.append(ScriptResult(file="scripts/demo/quick.py", + status=Status.PASSED, + duration_seconds=1.5, cap_seconds=300, + exit_code=0)) + report.results.append(ScriptResult(file="scripts/demo/slow.py", + status=Status.TIMEOUT, + duration_seconds=300.4, cap_seconds=300)) + report.results.append(ScriptResult(file="scripts/demo/gui.py", + status=Status.SKIPPED, + skip_reason="GUI script")) + report.write(tmp_path / "reports") + + text = summary.read_text() + rows = [line for line in text.splitlines() if line.startswith("| `")] + assert "slow.py" in rows[0] + assert "quick.py" in rows[1] + # Untimed entries sort last and show no fabricated duration. + assert "gui.py" in rows[2] + assert "| — |" in rows[2] + # The cap is shown for the entry it bound, not on every row. + assert rows[0].endswith("| 300s |") + assert rows[1].endswith("| |") + assert "3 entries" in text + assert "301.9s total" in text + + +def test_step_summary_is_appended_by_each_leg(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + scripts = _report() + scripts.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=1.0, cap_seconds=300, + exit_code=0)) + scripts.write(tmp_path / "reports") + + notebooks = _report(directory="notebooks/demo", run_type="notebook") + notebooks.results.append(ScriptResult(file="notebooks/demo/alpha.ipynb", + status=Status.PASSED, + duration_seconds=2.0, cap_seconds=300, + exit_code=0)) + notebooks.write(tmp_path / "reports") + + text = summary.read_text() + assert text.count("### Smoke timings") == 2 + assert "scripts/demo" in text and "notebooks/demo" in text + + +def test_no_step_summary_off_ci(tmp_path, monkeypatch): + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + report = _report() + report.results.append(ScriptResult(file="scripts/demo/alpha.py", + status=Status.PASSED, + duration_seconds=1.0, cap_seconds=300, + exit_code=0)) + assert report.append_step_summary() is False + + +def test_unwritable_step_summary_does_not_fail_the_run(tmp_path, monkeypatch): + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "nope" / "s.md")) + report = _report() + assert report.append_step_summary() is False + + +# --- the runner records the cap and exit code (build_util) -------------------- + + +def test_execute_script_records_cap_and_exit_code(tmp_path, monkeypatch): + import build_util + + monkeypatch.chdir(tmp_path) + script = tmp_path / "ok.py" + script.write_text("print('hi')\n") + + report = _report() + env = dict(os.environ, BUILD_SCRIPT_TIMEOUT="77") + build_util.execute_script(str(script), report=report, env=env) + + result = report.results[0] + assert result.status == Status.PASSED + assert result.cap_seconds == 77 + assert result.exit_code == 0 + assert result.to_timings_entry()["cap_s"] == 77 + + +def test_execute_script_records_the_failing_exit_code(tmp_path, monkeypatch): + import build_util + + monkeypatch.chdir(tmp_path) + script = tmp_path / "bad.py" + script.write_text("import sys; sys.exit(3)\n") + + report = _report() + build_util.execute_script(str(script), report=report, env=dict(os.environ)) + + result = report.results[0] + assert result.status == Status.FAILED + assert result.exit_code == 3 + assert result.cap_seconds == build_util.timeout_for(None) + + +# --- both report legs reach the same emission -------------------------------- + + +AUTOHANDS_DIR = Path(__file__).parent.parent / "autohands" + + +def _fake_workspace(tmp_path): + """A minimal workspace: an empty exclusion policy and a smoke profile.""" + build = tmp_path / "config" / "build" + build.mkdir(parents=True) + (build / "no_run.yaml").write_text("") + (build / "profile_smoke.yaml").write_text("defaults: {}\noverrides: []\n") + (tmp_path / "notebooks").mkdir() + (tmp_path / "scripts").mkdir() + return tmp_path + + +def _run_leg(workspace, script, directory): + import subprocess + + env = dict(os.environ, PYTHONPATH=str(AUTOHANDS_DIR)) + return subprocess.run( + [sys.executable, str(AUTOHANDS_DIR / script), "widgets", directory, + "--report-dir", "test-results"], + cwd=str(workspace), env=env, capture_output=True, text=True, + ) + + +def test_both_legs_emit_the_timings_file(tmp_path): + ws = _fake_workspace(tmp_path) + + scripts_leg = _run_leg(ws, "run_python.py", "scripts") + assert scripts_leg.returncode == 0, scripts_leg.stderr + notebooks_leg = _run_leg(ws, "run.py", "notebooks") + assert notebooks_leg.returncode == 0, notebooks_leg.stderr + + data = json.loads((ws / "test-results" / TIMINGS_FILENAME).read_text()) + assert {leg["run_type"] for leg in data["legs"]} == {"script", "notebook"} + # Both legs state the surface they measured, not "unknown". + assert {leg["env_profile"] for leg in data["legs"]} == {"profile_smoke.yaml"}