From 2ea413fda2216a2890d0db63f8f1884e5dfd2577 Mon Sep 17 00:00:00 2001 From: wms2537 Date: Tue, 8 Sep 2026 11:07:48 +0800 Subject: [PATCH] test: pinned-version controls must survive the bump they guard (#114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six of the seven controls anchored on the exact version in the tree: "bluenviron/mediamtx:1.20.1" "FROM node:24.20.0-bookworm-slim@" So a bump made the anchor vanish and the control reported VACUOUS — silencing itself on precisely the pull request that changes the thing it guards, which is the one moment it most needs to work. This is not hypothetical. It has happened twice already: the node 22 -> 24 bump silenced two controls earlier in this work, and Dependabot's MediaMTX 1.21.0 bump (#218) silenced another. Both times the fix was to hand-edit the anchor, which leaves the shape intact for next time. `Anchor` matches whatever version is pinned now and rewrites it to a sentinel, so a bump changes nothing about whether the control fires. Verified by simulating four simultaneous bumps — MediaMTX, node, Caddy, python — where one control previously went vacuous and now none do. Two smaller things the conversion forced, both worth keeping: * the runner now refuses a mutation that leaves the file byte-identical. A substitution that changes nothing tests nothing, however many times its anchor matched — and a pattern anchor makes that failure mode reachable in a way a literal one never was. * one control asserted on the guard's message INCLUDING a version number ("tells operators to install lap>=0.5"). That re-introduces the same fragility one layer along, so it now asserts on the sentence without the version. VACUOUS still fails the run. The design was right; the anchors were the part that could not survive contact with Dependabot. Signed-off-by: wms2537 --- CHANGELOG.md | 13 ++++++ scripts/test_pinned_versions.py | 78 ++++++++++++++++++++++++++------- 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b603a1..cd61feae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -395,6 +395,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Testing +- **The pinned-version controls no longer silence themselves on the pull request that bumps a + dependency.** Six of the seven anchored on the exact version in the tree — `mediamtx:1.20.1`, + `node:24.20.0-bookworm-slim@` — so a bump made the anchor vanish and the control reported VACUOUS. + That is the one moment the guard most needs to work, and it happened twice in practice: the node + 22 → 24 bump silenced two controls, and Dependabot's MediaMTX 1.21.0 bump silenced another. + + Anchors are now patterns that match whatever version is pinned and rewrite it to a sentinel, so a + bump changes nothing about whether the control fires. Verified by simulating four simultaneous + bumps (MediaMTX, node, Caddy, python): previously one control went vacuous, now none do. + + The runner also refuses a mutation that leaves the file byte-identical — a substitution that + changes nothing tests nothing, however many times its anchor matched. + - **A lost response followed by a retry is now proven to leave exactly one side effect** (#121), on a real route rather than a fake handler. The existing tests exercise the idempotency layer with a counting stub, which proves the layer dedupes — not that it is *mounted* where it matters. Its diff --git a/scripts/test_pinned_versions.py b/scripts/test_pinned_versions.py index 2f31740e..96f2abe0 100755 --- a/scripts/test_pinned_versions.py +++ b/scripts/test_pinned_versions.py @@ -9,9 +9,17 @@ The third case is the one that keeps the guard honest: if its regexes stop matching the files they read, it must SAY so rather than silently comparing nothing and passing. + +ANCHORS ARE PATTERNS, NOT LITERAL VERSIONS. Six of these seven controls used to anchor on the exact +version in the tree — `bluenviron/mediamtx:1.20.1`, `node:24.20.0-bookworm-slim@`. That makes a +control go VACUOUS on precisely the pull request that bumps its dependency, which is the one moment +the guard most needs to work: Dependabot's MediaMTX bump and the node 22 -> 24 bump each silenced +their own control this way. `Anchor` matches whatever version is there now and rewrites it to a +sentinel, so a bump changes nothing about whether the control fires. """ import os +import re import shutil import subprocess import sys @@ -19,33 +27,65 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) CHECK = os.path.join(ROOT, "scripts", "check_pinned_versions.py") +class Anchor: + """A version-shaped anchor: match whatever is pinned now, rewrite it to something else. + + A literal anchor names the version in the tree today, so the control silences itself the moment + that version changes — on the very pull request doing the changing. This matches the SHAPE and + substitutes a sentinel, so it keeps working across bumps without anybody remembering to edit it. + """ + + def __init__(self, pattern: str, replacement: str): + self.rx = re.compile(pattern) + self.replacement = replacement + + def count(self, src: str) -> int: + return len(self.rx.findall(src)) + + def apply(self, src: str) -> str: + return self.rx.sub(self.replacement, src) + + def __repr__(self) -> str: + return f"Anchor({self.rx.pattern!r})" + + +def apply_mutation(src: str, old, new: str) -> str: + return old.apply(src) if isinstance(old, Anchor) else src.replace(old, new) + + +def count_anchor(src: str, old) -> int: + return old.count(src) if isinstance(old, Anchor) else src.count(old) + + CASES = [ ( "the shipped bug: compose bumped, setup script left behind", "scripts/setup_caddy.sh", - 'VERSION="${CADDY_VERSION:-2.11.4}"', - 'VERSION="${CADDY_VERSION:-2.10.2}"', + Anchor(r'VERSION="\$\{CADDY_VERSION:-[\d.]+\}"', 'VERSION="${CADDY_VERSION:-0.0.1}"'), + None, "they disagree", ), ( "the two MediaMTX pins drifting apart", "deploy/compose.yml", - "bluenviron/mediamtx:1.20.1", - "bluenviron/mediamtx:1.20.0", + Anchor(r"bluenviron/mediamtx:[\d.]+", "bluenviron/mediamtx:0.0.1"), + None, "differs between the dev stack", ), ( "the drift merging #78 actually left: requirements moved, the recipe did not", "apps/ai/Dockerfile", - '"lap>=0.5.13"', - '"lap>=0.5"', - "tells operators to install lap>=0.5", + Anchor(r'"lap>=[\d.]+"', '"lap>=0.0.1"'), + None, + # Version-free on purpose: the sentinel this control writes is not a real version, and + # asserting on one would re-introduce the fragility the Anchor removed. + "tells operators to install lap>=", ), ( "the drift this check was written for: a base image bumped, the policy table left behind", "apps/ai/Dockerfile", - "FROM python:3.14.7-slim@", - "FROM python:3.15.0-slim@", + Anchor(r"FROM python:[\d.]+-slim@", "FROM python:0.0.1-slim@"), + None, "does not pin that image", ), ( @@ -58,8 +98,8 @@ ( "the shape #144 proposed: the builder image moved, CI left behind", "apps/web/Dockerfile", - "FROM node:24.20.0-bookworm-slim@", - "FROM node:26.8.1-bookworm-slim@", + Anchor(r"FROM node:[\d.]+-bookworm-slim@", "FROM node:0.0.1-bookworm-slim@"), + None, "different toolchains", ), ( @@ -76,8 +116,10 @@ ( "the guard's own parser drifting from the file it reads", "scripts/setup_caddy.sh", - 'VERSION="${CADDY_VERSION:-2.11.4}"', - 'VERSION="2.11.4"', + # Drops the ${CADDY_VERSION:-...} shape the guard's regex depends on, keeping whatever + # version is actually pinned — so this still exercises the parser after a Caddy bump. + Anchor(r'VERSION="\$\{CADDY_VERSION:-([\d.]+)\}"', r'VERSION="\1"'), + None, "parser has drifted", ), ] @@ -94,7 +136,7 @@ def main(): expect_n = case[5] if len(case) > 5 else None path = os.path.join(ROOT, rel) src = open(path).read() - n = src.count(old) + n = count_anchor(src, old) if n == 0 or (expect_n is not None and n != expect_n): print(f" VACUOUS {name}: anchor appears {n} times in {rel}" + (f" (expected {expect_n})" if expect_n is not None else "")) @@ -105,9 +147,15 @@ def main(): f"say how many you mean to change") bad += 1 continue + mutated = apply_mutation(src, old, new) + if mutated == src: + # A mutation that changes nothing tests nothing, however many times the anchor matched. + print(f" VACUOUS {name}: the mutation left {rel} byte-identical") + bad += 1 + continue shutil.copy(path, path + ".bak") try: - open(path, "w").write(src.replace(old, new)) + open(path, "w").write(mutated) r = run() ok = r.returncode == 1 and want in r.stdout print((" ok " if ok else " FAIL ") + name)