diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index bd4b95e..9eab28e 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -22,14 +22,21 @@ on: - "scripts/check-compat.py" - "scripts/compat-waivers.txt" - ".github/workflows/conformance.yml" + # NO `paths:` filter here, and that is the point. A path-filtered check + # cannot be a REQUIRED check: on a pull request that does not match, the + # context never reports at all, and branch protection waits for a report + # that will never come. The pull request hangs, permanently, with nothing + # to click. + # + # That is not hypothetical for this repo. PR #47 (a docs PR) only received + # these three checks because it happened to also touch conformance/run.py, + # and PR #48 — LICENSE and CONTRIBUTING.md — reported no checks at all. + # + # The push trigger above keeps its filter: nothing waits on a push, so + # skipping the corpus for a README typo on main costs nothing and saves a + # runner. On pull requests the corpus is ~40s, which is cheaper than one + # person wondering why their PR will not merge. pull_request: - paths: - - "schema/**" - - "tests/**" - - "conformance/**" - - "scripts/check-compat.py" - - "scripts/compat-waivers.txt" - - ".github/workflows/conformance.yml" workflow_dispatch: permissions: @@ -72,6 +79,27 @@ jobs: - name: run the corpus against the published schemas run: python3 conformance/run.py --junit conformance-report.xml + # What the corpus is WORTH, not just whether it is green. + # + # `run.py` answers "do the cases pass" — a corpus of zero cases answers + # that perfectly. `mutation_coverage.py` answers the harder question: + # delete a constraint from the schema and see whether any case notices. + # It has been in this repo, runnable, and wired into nothing, so the + # figure it produces could fall to zero between releases in silence. + # + # The floor is the CURRENT number rounded down (40.7% -> 40), not a + # target. That is the convention mutation-testing gates converge on + # (Stryker, mutmut): set the break threshold where you actually are, so + # it catches a regression on day one — an aspirational threshold goes red + # immediately and gets deleted by day three, which is worse than no gate. + # + # Deliberately NOT a ratchet. Ratchets exist to absorb run-to-run noise + # and there is none here: the run is deterministic — mutate the schema, + # replay the corpus, count. Raising the floor is a deliberate edit to + # this line, made when a corpus wave earns it. + - name: the corpus still kills the mutations it used to + run: python3 conformance/mutation_coverage.py --min-coverage 40 + # The corpus defines conformance; forge-cli is implementation #1 under # test, never the referee. Installing latest-stable here is deliberate: # it is how the Command Center installs it, so a divergence between the diff --git a/scripts/compat-waivers.txt b/scripts/compat-waivers.txt index a24446b..b00303d 100644 --- a/scripts/compat-waivers.txt +++ b/scripts/compat-waivers.txt @@ -25,10 +25,17 @@ # build.execution.notifications[0].retryOnFailure # 0.7.1: VALID 0.7.2: REJECTED (additionalProperties) # -# docs/releases/0.7.2.md says "100% backward compatible with 0.7.1" and -# "No breaking changes". That claim is false and should be corrected to name -# this change. This waiver exists so the gate can run green on history while -# the release note is fixed -- not to make the problem go away. +# docs/releases/0.7.2.md USED TO say "100% backward compatible with 0.7.1" and +# "No breaking changes". Both were false. The note was corrected in #46 and now +# opens by naming the break: "One breaking change, corrected here after the +# fact". The instruction this paragraph used to carry is therefore done, and +# leaving it standing would send the next reader to fix something already fixed. +# +# The waiver itself stays, permanently. 0.7.2 is published and its history +# cannot be rewritten, so the gate has to be told this break is known rather +# than new. What a waiver must never become is a way to stop the gate +# complaining about a break nobody dealt with -- this one was dealt with by +# correcting the release note, not by adding this line. 0.7.1->0.7.2 /$defs/notification # ---------------------------------------------------------------------------