diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 160aa976..903f0340 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -192,14 +192,19 @@ jobs: - name: Discard the committed snapshot run: rm -f drivers/*.lua - - name: The presence guard notices + # This is the fresh-clone path itself: drivers-present is what `make + # test`, `make e2e` and `make release` depend on, and on an empty + # drivers/ it fetches the snapshot rather than stopping to say how. + - name: The presence guard materialises the snapshot run: | - if make drivers-present 2>/dev/null; then - echo "::error::drivers-present passed with no drivers on disk" - exit 1 - fi - echo "guard fired as expected" + make drivers-present + want="$(jq -r '.drivers | length' drivers/BUNDLED_SOURCE.json)" + got="$(ls drivers/*.lua | wc -l | tr -d ' ')" + echo "guard fetched ${got} of ${want}" + [ "$want" = "$got" ] \ + || { echo "::error::guard left ${got} of ${want} drivers"; exit 1; } + # Idempotent by design: asking again writes the same bytes. - name: Fetch from the pin run: make drivers diff --git a/Makefile b/Makefile index 8a0e1b5f..220211f6 100644 --- a/Makefile +++ b/Makefile @@ -58,13 +58,24 @@ help: drivers: bash scripts/sync-bundled-drivers.sh -# Cheap enough to run before every test invocation, and it costs no network. -# Fetching here instead would put a remote call in the inner loop. +# Cheap enough to run before every test invocation: when the snapshot is on +# disk this costs one ls and no network, which is every run after the first. +# +# When it is not on disk the answer is to fetch it, not to stop and explain +# how. A fresh clone or worktree has an empty drivers/ -- they are gitignored +# -- and telling four people in a row to run one specific command is a worse +# use of their afternoon than running it for them. `go test` already downloads +# its modules on a fresh checkout; this is the same bargain, once. drivers-present: @ls drivers/*.lua >/dev/null 2>&1 || { \ - echo "drivers/ has no .lua files. Run 'make drivers' to fetch the" >&2; \ - echo "snapshot pinned in drivers/BUNDLED_SOURCE.json." >&2; \ - exit 1; } + echo "drivers/ is empty; fetching the snapshot pinned in drivers/BUNDLED_SOURCE.json"; \ + $(MAKE) --no-print-directory drivers || { \ + echo "" >&2; \ + echo "Could not fetch the bundled drivers. They come from" >&2; \ + echo "srcfl/device-drivers over the network and are not in git, so this" >&2; \ + echo "needs curl, jq and a route out. Fix that and run 'make drivers'," >&2; \ + echo "or copy drivers/*.lua from a checkout that already has them." >&2; \ + exit 1; }; } # ---- Testing ---- @@ -78,9 +89,11 @@ test: optimizer/.venv/.installed drivers-present cd go && FTW_TEST_OPTIMIZER_PYTHON=$(OPTIMIZER_PYTHON) go test ./internal/mpc \ -run 'TestExternalOptimizer(EndToEnd|PlansMultipleLoadpoints|PlansAndValidatesMultipleStorages)$$' +# The interpreter is chosen in the script, not here: the optimizer needs +# Python 3.11+ and PEP 660, and the python3 macOS ships is 3.9 with pip 21.2. +# PYTHON still overrides the choice. optimizer-install: - $(PYTHON) -m venv optimizer/.venv - optimizer/.venv/bin/pip install -e 'optimizer[test]' + PYTHON="$(PYTHON)" bash scripts/optimizer-venv.sh @touch optimizer/.venv/.installed optimizer-test: optimizer/.venv/.installed diff --git a/docs/development.md b/docs/development.md index d17e1d5e..908fbfb2 100644 --- a/docs/development.md +++ b/docs/development.md @@ -52,3 +52,13 @@ overrides are machine-specific and untracked. `bin/`, `dist/`, `artifacts/`, local databases, caches, `node_modules/` and `optimizer/.venv/` are disposable and ignored. Do not treat generated output or agent plans as project documentation. + +`drivers/*.lua` is ignored too: it is a snapshot of the commit pinned in +[`drivers/BUNDLED_SOURCE.json`](../drivers/BUNDLED_SOURCE.json), fetched, never +authored here. A fresh clone or `git worktree` therefore starts without either +it or the virtual environment, and the first `make test` builds both. There is +no separate setup step. Later runs cost nothing. + +The optimizer needs Python 3.11 or newer. `make` picks an interpreter that +qualifies, overridable with `PYTHON=`, and falls back to `uv` — optional, used +only when the machine has no suitable Python of its own. diff --git a/scripts/optimizer-venv.sh b/scripts/optimizer-venv.sh new file mode 100755 index 00000000..05475e57 --- /dev/null +++ b/scripts/optimizer-venv.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Build optimizer/.venv and install the optimizer into it. +# +# This is a script rather than two lines in the Makefile because choosing the +# interpreter is the whole job. The optimizer needs Python 3.11 or newer and a +# pip that can do a PEP 660 editable install. The python3 macOS ships is 3.9 +# with pip 21.2 and fails on both counts, and the error it prints -- "File +# setup.py or setup.cfg not found" -- reads like a packaging fault in this +# repository. It is not one. It is the wrong interpreter, and finding that out +# has cost several people an afternoon each. +# +# Order of preference: +# 1. $PYTHON -- the interpreter the old recipe used, so a box where that +# already worked keeps building exactly the venv it built before. +# 2. A python3.N on PATH, starting with the version the container image and +# CI use, so a local venv resolves the same wheels they do. +# 3. uv, which can fetch an interpreter when the machine has none. Optional +# throughout: it is used when nothing else works, never required. +# Nothing usable is an error that names both ways out. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PROJECT="${ROOT}/optimizer" +VENV="${PROJECT}/.venv" + +# The floor comes from the package itself, so this cannot drift away from it. +FLOOR="$(sed -n 's/^requires-python[[:space:]]*=[[:space:]]*">=\([0-9][0-9.]*\)".*/\1/p' \ + "${PROJECT}/pyproject.toml" | head -1)" +FLOOR="${FLOOR:-3.11}" +FLOOR_MAJOR="${FLOOR%%.*}" +FLOOR_MINOR="${FLOOR#*.}" + +# The version Dockerfile.optimizer and the CI optimizer job run. Preferred so a +# developer resolves the same wheels production does, but not required: any +# interpreter at or above the floor is accepted. +PREFERRED="3.12" + +satisfies_floor() { + local py="$1" + command -v "$py" >/dev/null 2>&1 || return 1 + "$py" -c "import sys; raise SystemExit(0 if sys.version_info[:2] >= (${FLOOR_MAJOR}, ${FLOOR_MINOR}) else 1)" \ + >/dev/null 2>&1 +} + +# A pip older than 21.3 has no PEP 660 support and fails the same way 3.9 does. +# An interpreter new enough for the floor normally ships a new enough pip; this +# is here so the promise holds on the one that does not. +pip_understands_editable() { + "${VENV}/bin/python" - <<'PY' >/dev/null 2>&1 +import sys +try: + from pip import __version__ as v +except Exception: + raise SystemExit(1) +major, minor = (int(part) for part in v.split(".")[:2]) +raise SystemExit(0 if (major, minor) >= (21, 3) else 1) +PY +} + +# A half-built venv is the normal state here, not an edge case: the failure +# this script exists to prevent leaves one behind, built on the interpreter +# that could not do the install. Keep the environment when its own interpreter +# qualifies -- reinstalling into it is what an unchanged machine wants -- and +# otherwise throw it away and choose again. +if [ -x "${VENV}/bin/python" ] && satisfies_floor "${VENV}/bin/python"; then + echo "optimizer: reusing .venv ($("${VENV}/bin/python" --version 2>&1))" +else + if [ -e "${VENV}" ]; then + echo "optimizer: replacing .venv, it has no python ${FLOOR}+" + rm -rf "${VENV}" + fi + + CHOSEN="" + for candidate in "${PYTHON:-}" "python${PREFERRED}" python3.13 python3.14 python3.11 python3; do + [ -n "${candidate}" ] || continue + if satisfies_floor "${candidate}"; then + CHOSEN="${candidate}" + break + fi + done + + if [ -n "${CHOSEN}" ]; then + echo "optimizer: building .venv with ${CHOSEN} ($("${CHOSEN}" --version 2>&1))" + "${CHOSEN}" -m venv "${VENV}" + elif command -v uv >/dev/null 2>&1; then + # uv only fetches the interpreter here. --seed puts pip in the result, so + # a venv built this way is indistinguishable from one built above and + # nothing downstream has to know which route it came by. + echo "optimizer: no python ${FLOOR}+ on PATH; building .venv with uv (python ${PREFERRED})" + uv venv --seed --python "${PREFERRED}" "${VENV}" + else + cat >&2 </dev/null || echo "not installed") + +Either install an interpreter: + + brew install python@${PREFERRED} # macOS + apt install python${PREFERRED}-venv # Debian/Ubuntu + +or install uv, which fetches one itself: + + curl -LsSf https://astral.sh/uv/install.sh | sh + +then run 'make optimizer-install' again. Core alone does not need this: the +optimizer is optional and 'cd go && go test ./...' runs without it. +MSG + exit 1 + fi +fi + +pip_understands_editable || "${VENV}/bin/python" -m pip install --quiet --upgrade pip +"${VENV}/bin/python" -m pip install -e "${PROJECT}[test]"