diff --git a/backend/docs/ci/README.md b/backend/docs/ci/README.md new file mode 100644 index 00000000..96730a58 --- /dev/null +++ b/backend/docs/ci/README.md @@ -0,0 +1,307 @@ +# backend/docs/ci — the proposed backend CI workflow + +`backend/docs/ci/backend.yml.proposed` is a ready-to-apply GitHub Actions +workflow for `backend/**`, and `backend/scripts/ci_local.sh` runs the same +steps in the same order locally. This file records what the proposal is, the +evidence for it, the measured local proof, and the decisions Sam still owns. + +## Why this exists + +`.github/workflows/server.yml` triggers only on `server/**`, so no workflow +runs the `backend/` tests (status-report/basescanning.json). A backend pull +request's only green checks today are the size label, CodeRabbit-skip and +Aikido — nothing executes a single backend test. + +Agents may not edit `.github/`, so the workflow ships as a reviewed proposal +instead of a commit. + +## What the workflow runs + +Two jobs: + +- **`backend-checks`** — install, ruff check, ruff format check, import + contracts, `make test` (with the four expired SD-96 gate test ids and the + cold-restart file deselected by name; see "Known failures"), `make + schemas`, `make validate`. Green on alpha as it stands; this is the job a + merge signal would require. +- **`backend-hygiene-known-red`** — `make hygiene` on its own, with + `continue-on-error: true`. The step is known red on alpha (see "Known + failures"), so it lives outside the required signal while staying visible; + the job name says what a failed annotation means. + +## How to apply + +Copy **one file**: + +```sh +cp backend/docs/ci/backend.yml.proposed .github/workflows/backend.yml +``` + +That is the whole change. The workflow that activates is the one proved +below. The applying pull request itself changes `.github/workflows/backend.yml`, +which is inside the workflow's own `paths` filter, so the first run happens on +that PR before anything merges. Nothing else is touched: no `.github/` edits, +no `uv.lock` or dependency pins, no thresholds or gate windows, no secrets. + +## How to run the same steps locally + +```sh +bash backend/scripts/ci_local.sh # fails fast: nonzero on the first failing gating step +bash backend/scripts/ci_local.sh --keep-going # runs every step, still exits nonzero if any gating step failed +``` + +The script mirrors the workflow job for job: the gating steps of +`backend-checks` first, the hygiene job last. The job's `actions/checkout` +and `astral-sh/setup-uv` steps are environment provisioning with no local +equivalent. Run it from a checkout with uv installed. + +The hygiene step runs through a soft wrapper that mirrors the job's +`continue-on-error: true`: the failure prints and the run still exits 0 when +only that step failed, exactly like the workflow. A gating-step failure fails +the run as it fails the job. Step timing uses a portable `now_ms` timer: +python3 milliseconds when python3 exists, whole-second timestamps otherwise — +GNU `date`'s `%N` is not available on macOS/BSD, where the team's shared Mac +runs (Codex P2 on PR #288). + +## Measured proof run + +Fresh clone of `SamGu-NRX/BaseScanning`, branch +`obv/burst11-bs-backend-ci-proposal` at the restructure head `5e51801a` +(`alpha` = `4bcbedba` plus this branch's commits), run with +`/bin/bash backend/scripts/ci_local.sh` on the packet's Debian 13 fleet +sandbox (8 cores, uv 0.13.1, Python 3.12; the uv cache was warm for this +run — the session's first sync had downloaded the ~2 GB set once, cold): + +| Step | Command | Result | Wall time | +|---|---|---|---| +| Install dependencies (locked) | `uv sync --frozen --extra api` | ok (warm cache; the session's cold sync downloaded ~2 GB once) | 0.0s | +| Ruff check | `uv run --frozen ruff check .` | ok | 0.0s | +| Ruff format check | `uv run --frozen ruff format --check .` | ok | 0.0s | +| Import contracts | `uv run --frozen lint-imports` | ok (5 kept, 0 broken) | 0.2s | +| Tests | `make test` with the deselections | ok — **1429 passed, 19 skipped, 4 deselected**, 8 warnings (1425 alpha + the 4 new regression tests) | 550.8s (pytest 548.72s) | +| Schemas | `make schemas` | ok | 0.3s | +| Committed fixtures | `make validate` | ok — scenes=5 verdicts=16 labels=61 packets=36 | 2.6s | +| Repo hygiene | `make hygiene` | **FAILED — named known failure, non-blocking** (mirrors the job's `continue-on-error`) | 0.4s | + +Start `11:01:58`, end `11:11:13` UTC. Overall script exit: **0** with the +closing line `gating steps green; the known-red hygiene step failed +non-blocking, as the mirrored job does on alpha` — the only failure is the +known-red hygiene step, which the mirrored job wraps in `continue-on-error`; +every gating step was green. The prior canonical run at the pre-restructure +head `528661c7` (hygiene as a required step, `--keep-going`, overall exit 1, +test step 527.3s / pytest 525.91s) is preserved in this file's history; its +per-step results match this run's. + +Baseline checks that the deselections target exactly the pre-existing +failures, re-verified at `5e51801a` on untouched alpha state: + +- `uv run --frozen --extra api pytest server/eval/tests/test_gates.py -q` + without any deselection: **4 failed, 90 passed in 6.92s** — the four + failures are exactly the ids listed below. +- `make hygiene`: fails with + `server/orchestrator/tests/data/packets/synthetic-meter.tar: deny-list + (synthetic-meter.tar)` — the tracked `.tar` the repo's own rules deny. +- `make test` on untouched alpha was measured at the prior head: 4 failed + (the four SD-96 ids), 1425 passed, 19 skipped, 14 collection errors (all + of `tests/test_capture_cold_restart.py`), 487.78s. + +Regressions: `backend/tests/test_ci_proposal_portability.py` pins both PR +#288 fixes and the Greptile P2 test gap. Four exit-semantics tests run the +real `ci_local.sh` end to end with stubbed `uv` and `make` first on PATH — +each stub exits with a code read from an env var keyed by its arguments and +logs every call — and pin: an all-green run executes every step in the +workflow's order; a failed gating step stops the default run and skips the +later steps; `--keep-going` runs the later steps and still exits nonzero; a +hygiene-only failure exits zero and says so. Those four fail against a +deliberately broken copy of the script that swallows every failure and always +exits 0 — it still printed `all steps green` after a required check failed — +and pass on the real script (shown on PR #288). The first four tests +(hygiene job shape; `%N` in the script; no `now_ms`) fail on the unfixed +tree and pass after. + +## Cache + +- `astral-sh/setup-uv` with `enable-cache: true` caches `~/.cache/uv`, keyed on + `backend/uv.lock` (`cache-dependency-glob: uv.lock`). Measured here: + **2.1 GB** for `~/.cache/uv`, **2.1 GB** for `backend/.venv`. +- `cache-python: true` also caches the uv-managed CPython 3.12, so the runner + does not re-download Python per run. +- First run on a cold runner pays the download once (~2 GB, dominated by the + torch pin in the default dependency groups); later runs restore from cache. +- The hygiene job runs the same cache. Its sync skips the `api` extra (the + hygiene script is stdlib-only), so it restores the same cache and adds no + new download. + +## Runner image + +`ubuntu-24.04` — the same pinned image `server.yml` uses. Justification: with +the `api` extra the suite is pure CPU (no GPU, no Modal, no docker, no Swift), +so a hosted x64 Linux runner is the cheapest sufficient image; pinning the +version (rather than `ubuntu-latest`) keeps the glibc/Python surface stable, +which matches this repository's existing convention. + +## Known failures on alpha (named, not hidden) + +At `alpha` = `4bcbedba`, three kinds of failures pre-exist this proposal. The +required job deselects the test ones by name and the hygiene one runs outside +the required signal; deleting the relevant block returns everything to CI. + +1. **Four expired SD-96 gate tests** — `server/eval/tests/test_gates.py`: + - `test_committed_fixtures_are_scored_against_their_labels` + - `test_offline_table_skips_the_engine_and_passes_the_unpatched_reference` + - `test_route_over_the_bar_is_known_until_expiry` + - `test_refresh_rewrites_pins_after_a_committed_edit` + + The SD-96 window closed `2026-10-04` in `backend/server/eval/gates.yaml` + (`positive_pass`, known_failure "conservative miss from near-wall + self-occlusion, SD-96", owner WS-09, until 2026-10-04). The window and the + thresholds are Sam's decision; this proposal does not touch `gates.yaml`. + +2. **14 collection errors in `tests/test_capture_cold_restart.py`** — the + file's tests all error at collection on alpha + (`scripts.replay_capture` no longer provides `load_receipt`) and are being + repaired in a separate change. The proposed job passes + `--ignore=tests/test_capture_cold_restart.py` with a comment; remove that + line when the repair lands on alpha. + +3. **Repo hygiene** — `make hygiene` fails on alpha: + `server/orchestrator/tests/data/packets/synthetic-meter.tar: deny-list + (synthetic-meter.tar)` — the hygiene script denies every tracked `.tar`, + and alpha tracks that file. The step runs in the separate + `backend-hygiene-known-red` job with `continue-on-error: true`; it turns + solid green when alpha stops tracking the file or the deny-list changes — + both outside this proposal — and the flag can then be deleted. + +### The known-red hygiene step: both restructurings, exact diff + +Codex P1 on this PR: do not ship a workflow with a permanently failing +required step. The failure itself cannot be fixed here: un-tracking the tar +or editing the deny-list is a `server/**`-side or history decision outside +this proposal. Two restructurings keep the default green: + +**Option 1 — a separate, clearly named non-blocking job (shipped default).** +The step keeps running on every backend change, its failure stays visible in +the run graph and annotations, and the pull-request check stays green: + +```diff + jobs: + backend-checks: + name: Backend checks + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + ... +- - name: Repo hygiene (make hygiene) +- run: make hygiene ++ ++ backend-hygiene-known-red: ++ name: Repo hygiene (known red on alpha) ++ continue-on-error: true ++ runs-on: ubuntu-24.04 ++ timeout-minutes: 15 ++ defaults: ++ run: ++ working-directory: backend ++ steps: ++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 ++ with: ++ persist-credentials: false ++ - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 ++ with: ++ working-directory: backend ++ enable-cache: true ++ cache-dependency-glob: uv.lock ++ cache-python: true ++ version: "0.13.1" ++ - name: Install dependencies (locked) ++ run: uv sync --frozen ++ - name: Repo hygiene (make hygiene) ++ run: make hygiene +``` + +Cost: a second runner pays a cache-restored sync for a step that fails in +about 0.3s. Benefit: the denial keeps executing on every backend change, so +the moment alpha stops tracking the file, the job turns solid green on its +own and someone notices. + +**Option 2 — leave the step out of the workflow until it can pass** (the +reviewer's literal alternative). Delete the whole +`backend-hygiene-known-red` job block above. Greenest check list; the cost is +that the failing deny-list check runs nowhere until someone re-adds it, so +the regression stays invisible. + +Either way `backend-checks` is green on alpha. Option 1 ships. + +### The SD-96 decision (Sam's) + +Two options, with the exact diff between them: + +- **Option A — run them.** Delete the `PYTEST_ADDOPTS` env block from the + `Tests (make test)` step in `backend.yml.proposed` (and the matching + `PYTEST_ADDOPTS` line in `ci_local.sh`). The four tests run on every + backend pull request and fail until SD-96 is decided; the job is red from + day one and cannot gate merges; the failure set is exactly these four tests + plus, until the in-flight repair lands, the 14 cold-restart collection + errors: + + ```diff + - name: Tests (make test) + - env: + - PYTEST_ADDOPTS: >- + - --deselect server/eval/tests/test_gates.py::test_committed_fixtures_are_scored_against_their_labels + - --deselect server/eval/tests/test_gates.py::test_offline_table_skips_the_engine_and_passes_the_unpatched_reference + - --deselect server/eval/tests/test_gates.py::test_route_over_the_bar_is_known_until_expiry + - --deselect server/eval/tests/test_gates.py::test_refresh_rewrites_pins_after_a_committed_edit + - --ignore=tests/test_capture_cold_restart.py + run: make test + ``` + +- **Option B — deselect those four test ids by name (as proposed).** The job + is green; the other gate tests in `test_gates.py` still run (the + deselection is by exact test id, so the blast radius is exactly four + tests); restoring them later is deleting one env block, and the block's + comment plus this README point at the SD-96 decision. This is the shipped + default and the diff above is what Option A would apply on top of it. + +**Recommendation: Option B.** A job that is red from day one trains everyone +to ignore it and can never gate a merge — it re-creates the current situation +(green checks that mean nothing) one level down. The deselection is auditable +(by name, with a comment pointing at the decision) and reversible in one +deletion; the cost is that exactly those four tests leave CI coverage until +restored, which is the honest price of an expired window Sam has not decided +yet. Reverse this call the moment SD-96 is decided: delete the block and the +tests come back. + +## What the job deliberately excludes + +- **GPU.** The job syncs `--extra api` only. `--all-extras` would install the + GPU torch pin and change OWLv2 parity behavior (backend/Makefile comment); + no GPU runner, no CUDA. +- **Modal.** No `modal` invocations: no `make weights` (Modal weights fetch), + no `make e2e-dev` (Modal capture-api), no cloud state/storage backends. The + in-suite journeys run no-cloud by the documented SD-73/SD-11 settings + (`HSS_ORCH_STAGE_STUBS=1`, prewarm off). +- **Fixtures fetch.** No `make fixtures` — the store-backed families (ETH3D + laser PLYs and friends) are unreachable from CI; the job relies on the + committed fixtures (`make validate` covers those: scenes=5, verdicts=16, + labels=61, packets=36). +- **Live e2e.** No `make e2e` / `e2e-local` (needs Modal pipeline outputs — + the recon `points.ply` files are gitignored Modal outputs), no + `validate-full`, no `criteria-golden` (needs the laser PLYs from + `make fixtures`), no `make evals` (a stage that cannot run FAILS by design), + no `reads-extra`/`lamar`. The iOS suite and web checks are other workflows' + scope. + +## Not verified here + +- Actual execution on a GitHub-hosted runner (applying the workflow requires + a `.github/` change — that is Sam's copy step). The workflow text, its pins + and its inputs are verified against the actions' published `action.yml` at + the pinned tags, and both pins are the exact commits `server.yml` already + runs. +- Hosted-runner wall times (see the caveat under the measured table). +- macOS hardware: `ci_local.sh` portability is verified against a BSD-like + `date` stub (a `%N`-less date, and a run with no `python3` on PATH) plus + code inspection (no `%N`, no bash-4-only constructs, `set -euo pipefail` + and arithmetic that work on the stock `/bin/bash` 3.2), not on a real Mac. +- The behavior of the four SD-96 tests after Sam's decision. diff --git a/backend/docs/ci/backend.yml.proposed b/backend/docs/ci/backend.yml.proposed new file mode 100644 index 00000000..d21fbb3c --- /dev/null +++ b/backend/docs/ci/backend.yml.proposed @@ -0,0 +1,152 @@ +# PROPOSED backend CI workflow — not installed. +# +# Why: .github/workflows/server.yml triggers only on server/**, so no workflow +# runs the backend/ tests (status-report/basescanning.json; a backend pull +# request's only green checks today are the size label, CodeRabbit-skip and +# Aikido). Agents may not edit .github/, so the workflow ships as a reviewed +# proposal. +# +# To apply: copy this file to .github/workflows/backend.yml. That is the only +# change needed. backend/scripts/ci_local.sh runs both jobs' steps in the +# same order locally, and backend/docs/ci/README.md records the measured run +# this proposal was proved with. +# +# Conventions mirror .github/workflows/server.yml: actions pinned by commit SHA +# with the verified tag in a comment, ubuntu-24.04, contents:read, concurrency +# group that cancels superseded pull-request runs. + +name: backend + +on: + push: + branches: [main] + paths: + - "backend/**" + - ".github/workflows/backend.yml" + pull_request: + paths: + - "backend/**" + - ".github/workflows/backend.yml" + +permissions: + contents: read + +concurrency: + group: backend-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + backend-checks: + name: Backend checks + # Hosted x64 Linux is enough: with the api extra the suite is pure CPU — + # no GPU, no Modal, no docker, no Swift (see backend/docs/ci/README.md). + # The pinned image matches server.yml. + runs-on: ubuntu-24.04 + timeout-minutes: 30 + defaults: + run: + working-directory: backend + steps: + # v7.0.1 — the exact commit server.yml already runs on every change. + # Tag verified to exist (raw action.yml at v7.0.1). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # v10.2.0 — the exact commit server.yml already runs on every change. + # Tag verified to exist (raw action.yml at v10.2.0; inputs enable-cache, + # cache-dependency-glob, cache-python and working-directory confirmed + # there). Caches ~/.cache/uv keyed on backend/uv.lock, so the heavy + # first install (~2 GB with the api extra) is paid once. + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + working-directory: backend + enable-cache: true + cache-dependency-glob: uv.lock + cache-python: true + version: "0.13.1" # the uv version the local proof ran with + + - name: Install dependencies (locked) + run: uv sync --frozen --extra api + + - name: Ruff check + run: uv run --frozen ruff check . + + - name: Ruff format check + run: uv run --frozen ruff format --check . + + - name: Import contracts (lint-imports) + run: uv run --frozen lint-imports + + - name: Tests (make test) + env: + # Known failures, deselected by name so this job is a usable green + # signal from day one. Deleting this env block returns all of them + # to CI and the job goes red again — see backend/docs/ci/README.md + # ("Known failures") for the exact effect of both options. + # + # 1. Four expired SD-96 gate tests. The SD-96 window closed + # 2026-10-04 in backend/server/eval/gates.yaml (positive_pass, + # known_failure "conservative miss from near-wall self-occlusion, + # SD-96"). The window and thresholds are Sam's decision; nothing + # here touches gates.yaml. + # 2. tests/test_capture_cold_restart.py errors at collection on + # alpha (scripts.replay_capture no longer provides load_receipt) + # and is being repaired in a separate change. Remove this + # --ignore line when that repair lands on alpha. + PYTEST_ADDOPTS: >- + --deselect server/eval/tests/test_gates.py::test_committed_fixtures_are_scored_against_their_labels + --deselect server/eval/tests/test_gates.py::test_offline_table_skips_the_engine_and_passes_the_unpatched_reference + --deselect server/eval/tests/test_gates.py::test_route_over_the_bar_is_known_until_expiry + --deselect server/eval/tests/test_gates.py::test_refresh_rewrites_pins_after_a_committed_edit + --ignore=tests/test_capture_cold_restart.py + run: make test + + - name: Schemas (make schemas) + run: make schemas + + + - name: Committed fixtures (make validate) + run: make validate + + # The known-red step lives in its own clearly named job so it cannot fail + # the required signal (Codex P1 on PR #288: do not ship a required step + # that always fails). make hygiene is red on alpha at 4bcbedba, + # pre-existing: scripts/check_repo_hygiene.py denies every tracked .tar, + # and alpha tracks + # server/orchestrator/tests/data/packets/synthetic-meter.tar. Fixing that + # means un-tracking the file or changing the deny-list — both outside this + # proposal. continue-on-error keeps the pull-request check green while the + # step's failure stays visible in the run graph and its annotations. The + # job goes solid green when alpha stops tracking that file or the + # deny-list changes, and the flag can then be deleted. + # backend/docs/ci/README.md documents this and the alternative (leaving + # the step out of the workflow until it can pass) with the exact diff. + backend-hygiene-known-red: + name: Repo hygiene (known red on alpha) + continue-on-error: true + runs-on: ubuntu-24.04 + timeout-minutes: 15 + defaults: + run: + working-directory: backend + steps: + # Same pins as backend-checks; see the comment there. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + working-directory: backend + enable-cache: true + cache-dependency-glob: uv.lock + cache-python: true + version: "0.13.1" + + # No api extra needed: check_repo_hygiene.py is stdlib-only (re, + # subprocess, sys, pathlib), so the default locked set is enough. + - name: Install dependencies (locked) + run: uv sync --frozen + + - name: Repo hygiene (make hygiene) + run: make hygiene diff --git a/backend/scripts/ci_local.sh b/backend/scripts/ci_local.sh new file mode 100755 index 00000000..3d4a1d0c --- /dev/null +++ b/backend/scripts/ci_local.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# Local mirror of the proposed backend CI workflow +# (backend/docs/ci/backend.yml.proposed). +# +# Runs the steps of both jobs in the same order: the gating steps of +# backend-checks first, then the known-red hygiene job. The job's +# environment-provisioning steps (actions/checkout, astral-sh/setup-uv) have +# no local equivalent: run this from a checkout of the repository with uv +# installed (the proof ran with uv 0.13.1, Python 3.12). +# +# Usage: bash backend/scripts/ci_local.sh [--keep-going] (from anywhere; it +# cd's to backend/) +# +# default exits nonzero on the FIRST failing gating step, mirroring +# the job, where a failed step fails the job immediately. +# --keep-going runs every step anyway, collecting each step's result, and +# still exits nonzero when any gating step failed — one +# command for the full per-step report backend/docs/ci/README.md +# documents. +# +# make hygiene is known red on alpha, so the proposed workflow runs it in a +# separate job with continue-on-error: true (backend-hygiene-known-red). The +# local mirror does the same with step_soft: a failure there is printed and +# remembered, but the run still exits 0 when only that step failed, exactly +# like the job it mirrors. Nothing is hidden; the step line says FAILED. +set -euo pipefail + +cd "$(dirname "$0")/.." # backend/ + +KEEP_GOING=0 +FAILED=0 +SOFT_FAILED=0 +if [ "${1:-}" = "--keep-going" ]; then + KEEP_GOING=1 +fi + +command -v uv >/dev/null 2>&1 || { + echo "==> [ci_local] uv not found — install it first (https://docs.astral.sh/uv/)" >&2 + exit 1 +} + +# Millisecond wall clock. GNU date's %N is not portable (macOS/BSD date +# prints a literal 'N' and the arithmetic below would abort the script — +# Codex P2 on PR #288), so the time comes from python3 when it exists and +# falls back to whole-second timestamps: every POSIX date has %s, and the +# value stays in milliseconds. +now_ms() { + if command -v python3 >/dev/null 2>&1; then + python3 -c 'import time; print(int(time.time() * 1000))' + else + date +%s000 + fi +} + +step() { + local name="$1" + shift + local t0 ms + t0=$(now_ms) + printf '==> [ci_local] %s\n' "$name" + if "$@"; then + ms=$(($(now_ms) - t0)) + printf '<== [ci_local] %s ok in %d.%ds\n' "$name" $((ms / 1000)) $(((ms % 1000) / 100)) + else + ms=$(($(now_ms) - t0)) + printf '<== [ci_local] %s FAILED after %d.%ds\n' "$name" $((ms / 1000)) $(((ms % 1000) / 100)) >&2 + if [ "$KEEP_GOING" -eq 1 ]; then + FAILED=1 + return 0 + fi + exit 1 + fi +} + +# Mirrors a step that runs with continue-on-error: true in +# backend.yml.proposed: a failure is reported and stays visible but cannot +# fail the run. +step_soft() { + local name="$1" + shift + local t0 ms + t0=$(now_ms) + printf '==> [ci_local] %s\n' "$name" + if "$@"; then + ms=$(($(now_ms) - t0)) + printf '<== [ci_local] %s ok in %d.%ds\n' "$name" $((ms / 1000)) $(((ms % 1000) / 100)) + else + ms=$(($(now_ms) - t0)) + printf '<== [ci_local] %s FAILED (non-blocking, mirrors the job) after %d.%ds\n' "$name" $((ms / 1000)) $(((ms % 1000) / 100)) >&2 + SOFT_FAILED=1 + fi +} + +step "uv sync --frozen --extra api" uv sync --frozen --extra api +step "ruff check" uv run --frozen ruff check . +step "ruff format --check" uv run --frozen ruff format --check . +step "lint-imports" uv run --frozen lint-imports + +# Known failures, deselected by name so ci_local mirrors the proposed job — +# see the comment in backend.yml.proposed and backend/docs/ci/README.md +# ("Known failures"). +step "make test (known failures deselected)" env \ + PYTEST_ADDOPTS='--deselect server/eval/tests/test_gates.py::test_committed_fixtures_are_scored_against_their_labels --deselect server/eval/tests/test_gates.py::test_offline_table_skips_the_engine_and_passes_the_unpatched_reference --deselect server/eval/tests/test_gates.py::test_route_over_the_bar_is_known_until_expiry --deselect server/eval/tests/test_gates.py::test_refresh_rewrites_pins_after_a_committed_edit --ignore=tests/test_capture_cold_restart.py' \ + make test + +step "make schemas" make schemas +step "make validate" make validate + +# The hygiene job runs last, as in the workflow: known red on alpha, and +# non-blocking for the run, like its continue-on-error job. +step_soft "make hygiene (known red on alpha, non-blocking)" make hygiene + +if [ "$FAILED" -ne 0 ]; then + echo '[ci_local] FAILED — see the step results above' >&2 + exit 1 +fi +if [ "$SOFT_FAILED" -ne 0 ]; then + echo '[ci_local] gating steps green; the known-red hygiene step failed non-blocking, as the mirrored job does on alpha (backend/docs/ci/README.md)' + exit 0 +fi +echo '[ci_local] all steps green' diff --git a/backend/tests/test_ci_proposal_portability.py b/backend/tests/test_ci_proposal_portability.py new file mode 100644 index 00000000..0572ab51 --- /dev/null +++ b/backend/tests/test_ci_proposal_portability.py @@ -0,0 +1,222 @@ +"""Regressions for the proposed backend CI job and its local mirror. + +Pins the two PR #288 review findings: + +- The proposed workflow must not gate on `make hygiene`, which is known red + on alpha (the tracked synthetic-meter.tar is denied by the repo's own + hygiene rules). The step runs only in a job with continue-on-error: true. +- backend/scripts/ci_local.sh must time its steps without GNU date's %N, + which macOS/BSD date does not support; the script has to keep working there. +- backend/scripts/ci_local.sh's exit semantics, exercised end to end with + stubbed uv and make first on PATH (Greptile P2): a failed gating step stops + the default run and skips the later steps, --keep-going runs the later + steps and still exits nonzero, and a hygiene-only failure exits zero and + says so. Without these, a change that reports success after a required + check fails would still pass the suite. +""" + +import os +import re +import subprocess +import tempfile +from pathlib import Path + +import yaml + +BACKEND = Path(__file__).resolve().parents[1] +PROPOSAL = BACKEND / "docs" / "ci" / "backend.yml.proposed" +CI_LOCAL = BACKEND / "scripts" / "ci_local.sh" + +BSD_DATE_SHIM = """\ +#!/bin/sh +# A BSD-like date stub: %s works, %N is not a conversion and prints +# literally. Absolute binary paths, so it works with an empty PATH. +fmt="${1:-}" +case "$fmt" in + +*) ;; + *) exec /usr/bin/date "$@" ;; +esac +epoch=$(/usr/bin/date +%s) +body="${fmt#+}" +printf '%s\\n' "$(printf '%s' "$body" | /usr/bin/sed "s/%s/$epoch/")" +""" + + +def _workflow_jobs(): + workflow = yaml.safe_load(PROPOSAL.read_text()) + assert workflow and "jobs" in workflow, "backend.yml.proposed must define jobs" + return workflow["jobs"] + + +def _jobs_running_hygiene(jobs): + return [ + job_id + for job_id, job in jobs.items() + if any("make hygiene" in (step.get("run") or "") for step in job.get("steps") or []) + ] + + +def test_hygiene_step_cannot_fail_a_required_job(): + """The known-red hygiene step runs only under continue-on-error: true.""" + jobs = _workflow_jobs() + hygiene_jobs = _jobs_running_hygiene(jobs) + assert hygiene_jobs, "the workflow must keep running make hygiene somewhere" + for job_id in hygiene_jobs: + assert jobs[job_id].get("continue-on-error") is True, ( + f"job {job_id} runs make hygiene without continue-on-error: true" + ) + assert "backend-checks" in jobs, "the required backend-checks job must exist" + assert "backend-checks" not in hygiene_jobs, ( + "backend-checks must not gate on make hygiene: the step is known red on alpha" + ) + + +def test_script_never_calls_date_with_percent_n(): + """GNU %N renders as a literal N on macOS/BSD and aborts the arithmetic.""" + code_lines = [line for line in CI_LOCAL.read_text().splitlines() if not line.lstrip().startswith("#")] + offenders = [line for line in code_lines if re.search(r"date\s+\+[^|;&)\n]*%[0-9]*N", line)] + assert not offenders, f"ci_local.sh must not use GNU date %N: {offenders}" + + +def _now_ms_output(path_env: str) -> str: + text = CI_LOCAL.read_text() + match = re.search(r"^now_ms\(\) \{.*?^\}", text, re.M | re.S) + assert match, "now_ms() not found in scripts/ci_local.sh" + result = subprocess.run( + ["/bin/bash", "-c", match.group(0) + "\nnow_ms"], + capture_output=True, + text=True, + check=True, + env={**os.environ, "PATH": path_env}, + ) + return result.stdout.strip() + + +def _install_bsd_date_shim(tmp: str) -> str: + shim = Path(tmp) / "date" + shim.write_text(BSD_DATE_SHIM) + shim.chmod(0o755) + return tmp + + +def test_timer_works_when_date_lacks_percent_n(): + """python3-backed timer under a BSD-like date stub returns an integer.""" + with tempfile.TemporaryDirectory() as tmp: + out = _now_ms_output(f"{_install_bsd_date_shim(tmp)}:{os.environ['PATH']}") + assert out.isdigit(), f"now_ms printed a non-integer: {out!r}" + + +# --- Greptile P2 on PR #288: the script's exit semantics, exercised --------- + +# One stub source serves both tools: the recorded name comes from $0 at run +# time. It appends one "tool args..." line per call to $CI_STUB_LOG so tests +# can assert which steps ran and in what order, and exits with the code named +# by an env var keyed by the call's arguments: `make test` looks up +# CI_STUB_MAKE_TEST, `uv sync --frozen --extra api` looks up +# CI_STUB_UV_SYNC___FROZEN___EXTRA_API — uppercased, every non-alphanumeric +# character folded to _. A key that is not set exits 0 unless CI_STUB_DEFAULT +# says otherwise. +STUB_SOURCE = r"""#!/bin/sh +log="${CI_STUB_LOG:?CI_STUB_LOG must be set}" +printf '%s %s\n' "$(basename "$0")" "$*" >>"$log" +key=$(printf '%s %s' "$(basename "$0")" "$*" \ + | LC_ALL=C tr '[:lower:]' '[:upper:]' | LC_ALL=C tr -c '[:alnum:]' '_') +code=$(eval "printf '%s' \"\${CI_STUB_${key}:-}\"") +[ -n "$code" ] || code="${CI_STUB_DEFAULT:-0}" +exit "$code" +""" + +# The steps backend/scripts/ci_local.sh runs, in the proposed workflow's +# order; the stubs make each one a recorded, sub-second call. +STEPS_IN_ORDER = [ + "uv sync --frozen --extra api", + "uv run --frozen ruff check .", + "uv run --frozen ruff format --check .", + "uv run --frozen lint-imports", + "make test", + "make schemas", + "make validate", + "make hygiene", +] + + +def _stub_key(call: str) -> str: + """The env-var key a stub call looks up, matching the shell stub's folding.""" + return re.sub(r"[^A-Z0-9]", "_", call.upper()) + + +def _run_ci_local(codes: dict[str, int], keep_going: bool = False): + """Run the real scripts/ci_local.sh with stubbed uv and make first on PATH. + + `codes` maps a call's argument string ("make hygiene") to the exit code + its stub must return. Returns (proc, calls): the run's result and the + recorded stub calls, one line per call, in call order. + """ + with tempfile.TemporaryDirectory() as tmp: + stub_bin = Path(tmp) / "bin" + stub_bin.mkdir() + log = Path(tmp) / "calls.log" + for tool in ("uv", "make"): + stub = stub_bin / tool + stub.write_text(STUB_SOURCE) + stub.chmod(0o755) + env = { + **os.environ, + "PATH": f"{stub_bin}{os.pathsep}{os.environ['PATH']}", + "CI_STUB_LOG": str(log), + } + for call, code in codes.items(): + env[f"CI_STUB_{_stub_key(call)}"] = str(code) + args = ["/bin/bash", str(CI_LOCAL)] + if keep_going: + args.append("--keep-going") + proc = subprocess.run(args, capture_output=True, text=True, env=env, timeout=300) + calls = log.read_text().splitlines() + return proc, calls + + +def test_all_green_run_exits_zero_and_runs_every_step_in_order(): + """Nothing fails: every step runs once, in the workflow's order, exit 0.""" + proc, calls = _run_ci_local({}) + assert proc.returncode == 0, f"exit {proc.returncode}: {proc.stderr}" + assert calls == STEPS_IN_ORDER + assert "all steps green" in proc.stdout + + +def test_default_run_stops_at_the_first_failing_gating_step(): + """A failed gating step fails the default run and skips the later steps.""" + proc, calls = _run_ci_local({"uv run --frozen ruff check .": 3}) + assert proc.returncode != 0, "a failed gating step must fail the default run" + assert "uv run --frozen ruff check ." in calls + for skipped in STEPS_IN_ORDER[2:]: + assert skipped not in calls, f"the failing step must stop the run: {skipped} ran" + assert "ruff check" in proc.stderr and "FAILED" in proc.stderr + + +def test_keep_going_runs_later_steps_and_still_exits_nonzero(): + """--keep-going runs the later steps and still exits nonzero.""" + proc, calls = _run_ci_local({"uv run --frozen ruff check .": 3}, keep_going=True) + assert proc.returncode != 0, "--keep-going still exits nonzero on a gating failure" + assert "uv run --frozen ruff check ." in calls + for step in STEPS_IN_ORDER[2:]: + assert step in calls, f"--keep-going must run the later steps: {step}" + assert calls.index("uv run --frozen ruff check .") < calls.index("make schemas") + assert "[ci_local] FAILED" in proc.stderr + + +def test_hygiene_only_failure_exits_zero_and_says_so(): + """The known-red hygiene step fails non-blocking: exit 0, failure visible.""" + proc, calls = _run_ci_local({"make hygiene": 2}) + assert proc.returncode == 0, f"hygiene-only failure must exit 0: {proc.stderr}" + assert "make hygiene" in calls + assert "FAILED" in proc.stderr, "the hygiene failure must be printed, not hidden" + assert "gating steps green" in proc.stdout + + +def test_timer_falls_back_to_whole_seconds_without_python3(): + """No python3 on PATH: the whole-second date fallback still yields an integer.""" + with tempfile.TemporaryDirectory() as tmp: + out = _now_ms_output(_install_bsd_date_shim(tmp)) + assert out.isdigit() and int(out) % 1000 == 0, ( + f"the no-python3 fallback must return whole-second milliseconds: {out!r}" + )