From 20b431a33ed3dc584a2d5cefc09f0797b442fdf3 Mon Sep 17 00:00:00 2001 From: Param Harrison Date: Sun, 1 Nov 2026 15:40:18 +0200 Subject: [PATCH 1/5] Test: factory-04 solution runs the lab's own hint command (expected red) Co-Authored-By: Claude Sonnet 5.5 --- tests/structure/lab-artifacts.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/structure/lab-artifacts.ts b/tests/structure/lab-artifacts.ts index eedcbbb..85c3d56 100644 --- a/tests/structure/lab-artifacts.ts +++ b/tests/structure/lab-artifacts.ts @@ -15,7 +15,7 @@ const file = (path: string, body: string) => `cat > ${path} <<'EOF'\n${body}\nEO export const NEEDS: Record = { "daily-02": ["api"], "factory-03": [".factory/config.json", "factory/main.ts"], - "factory-04": ["tests/contract/sse-events.test.ts"], + "factory-04": ["factory/engine/workflows.test.ts"], "advanced-05": [".claude/settings.json"], }; @@ -84,9 +84,10 @@ export const SOLUTIONS: Record = { `printf 'Given a university prompt\\nWhen the agent builds the worksheet\\nThen the worksheet band is university\\nGiven an empty prompt\\nWhen the agent builds the worksheet\\nThen the api returns status 400\\n' > ${W}/specs/1/acceptance.md`, ), "factory-03": sh(`mkdir -p ${W}`, `make factory ISSUE=1 FLAGS=--dry-run > ${W}/dry-run.txt`), + // The command the lab's own hint gives, so the proof is checked against the output a learner gets. "factory-04": sh( `mkdir -p ${W}`, - `printf ' ✓ tests/contract/sse-events.test.ts (2 tests) 4ms\\n Test Files 1 passed (1)\\n Tests 2 passed (2)\\n' > ${W}/contract.log`, + `npx vitest run factory/engine/workflows.test.ts > ${W}/contract.log 2>&1`, ), "advanced-01": sh( `mkdir -p ${W}/agents`, From 183b733d007c0209b4ac250ea1021b1917c5c15e Mon Sep 17 00:00:00 2001 From: Param Harrison Date: Sun, 1 Nov 2026 15:43:40 +0200 Subject: [PATCH 2/5] Add factory labs 05 to 07 and fix factory-04 proof so its own hint passes Co-Authored-By: Claude Sonnet 5.5 --- docs/README.md | 5 +- docs/labs/README.md | 3 ++ docs/labs/walkthrough-factory.md | 44 ++++++++++++++++- labs/factory/04-run-the-contract.md | 2 +- labs/factory/05-run-the-checks.md | 62 ++++++++++++++++++++++++ labs/factory/06-count-the-votes.md | 60 +++++++++++++++++++++++ labs/factory/07-trip-a-band.md | 74 +++++++++++++++++++++++++++++ tests/structure/lab-artifacts.ts | 22 +++++++++ 8 files changed, 269 insertions(+), 3 deletions(-) create mode 100644 labs/factory/05-run-the-checks.md create mode 100644 labs/factory/06-count-the-votes.md create mode 100644 labs/factory/07-trip-a-band.md diff --git a/docs/README.md b/docs/README.md index 02d46a3..9d4f152 100644 --- a/docs/README.md +++ b/docs/README.md @@ -81,9 +81,12 @@ Which lab goes with which module: | `factory-02` | [F2 Design](factory/f2-design.md) | | `factory-03` | [F3 Build](factory/f3-build.md) | | `factory-04` | [F8 Workflows](factory/f8-workflows.md) | +| `factory-05` | [F4 Test](factory/f4-test.md) | +| `factory-06` | [F5 Review](factory/f5-review.md) | +| `factory-07` | [F7 Maintain](factory/f7-maintain.md) | | `advanced-05` | [F11 Cloud sessions](factory/f11-cloud-sessions.md) | -Modules with no lab yet are read-only: F0, F4 to F7, F9, F10, F12 and F13. +Modules with no lab yet are read-only: F0, F6, F9, F10, F12 and F13. ## 4. Cloud sessions diff --git a/docs/labs/README.md b/docs/labs/README.md index f4b8750..fcc3adc 100644 --- a/docs/labs/README.md +++ b/docs/labs/README.md @@ -60,6 +60,9 @@ Each step says where to type it: "in your shell", "in Claude Code" or "in your e | `factory-02` | factory | Write acceptance before code | [F2 Design](../factory/f2-design.md) | | `factory-03` | factory | See what a run would do | [F3 Build](../factory/f3-build.md) | | `factory-04` | factory | Check the workflow contract | [F8 Workflows](../factory/f8-workflows.md) | +| `factory-05` | factory | Run the checks that stop a bad test | [F4 Test](../factory/f4-test.md) | +| `factory-06` | factory | Count the votes behind a review | [F5 Review](../factory/f5-review.md) | +| `factory-07` | factory | Trip a band and read the intent it writes | [F7 Maintain](../factory/f7-maintain.md) | | `advanced-01` | advanced | Write a read-only reviewer subagent | [execution](../levels/execution.md) | | `advanced-02` | advanced | Measure what a call costs | [agent SDK](../agent/agent-sdk.md) | | `advanced-03` | advanced | Work in a parallel worktree | [agent teams](../electives/agent-teams.md) | diff --git a/docs/labs/walkthrough-factory.md b/docs/labs/walkthrough-factory.md index c2b661c..177b73e 100644 --- a/docs/labs/walkthrough-factory.md +++ b/docs/labs/walkthrough-factory.md @@ -158,6 +158,48 @@ The end of `contract.log`: The test loads every file in `.factory/workflows/`, so one bad workflow fails it by name. Change a stage in one file, rerun, and read which workflow broke. +## factory-05: Run the checks that stop a bad test + +**Pairs with** [F4 Test](../factory/f4-test.md) + +**The problem.** "The tests pass" is the model's word until a check reproduces it. The checks (plan-drift, holdout, mutation, manifest-deps) are code with tests of their own. + +```sh +make lab ID=factory-05 +mkdir -p .labs/work +npx vitest run factory/engine/checks > .labs/work/checks.log 2>&1 +grep '✓' .labs/work/checks.log +make lab ID=factory-05 +``` + +**What I got.** The log lists `manifest-deps.test.ts` (23 tests), `mutation.test.ts` (3), `holdout.test.ts` (2) and `plan-drift.test.ts` (10), and ends with `Tests 38 passed (38)`. I ran the vitest command and read this output. I did not run the final green `make lab` here. + +## factory-06: Count the votes behind a review + +**Pairs with** [F5 Review](../factory/f5-review.md) + +**The problem.** One agent's opinion is not a finding. The engine keeps a finding only when enough verifiers agree, and it caps nits and rounds. + +```sh +make lab ID=factory-06 +mkdir -p .labs/work +npx tsx -e 'import("./factory/engine/review.ts").then(m=>console.log("voters",m.VOTERS,"needed",m.NEEDED,"nits",m.MAX_NITS,"rounds",m.MAX_ROUNDS))' > .labs/work/review-rules.txt +npx vitest run factory/engine/review.test.ts > .labs/work/review.log 2>&1 +make lab ID=factory-06 +``` + +**What I got.** `review-rules.txt` holds `voters 3 needed 2 nits 3 rounds 3`, and `review.log` shows `factory/engine/review.test.ts (12 tests)` and `Tests 12 passed (12)`. The proof prints the constants again and compares, so a typed number fails. I did not run the final green `make lab` here. + +## factory-07: Trip a band and read the intent it writes + +**Pairs with** [F7 Maintain](../factory/f7-maintain.md) + +**The problem.** Maintain is arithmetic. A metric that leaves its band by three sigma writes the `intent.md` a new run starts from, with no model involved. + +You pick a number for `ci-failure-rate` against a fixed history of ten runs, then save the intent the factory writes for it. The lab's own proof scripts show how to print the sigma and the action. + +**What I got.** With the proof's history, a latest value of 30 gave `3 intent` and a latest value of 10 gave `0 none`. I ran both through `detect` and `actionFor`. I did not run the final green `make lab` here. + ## What this lane does not cover yet -The factory lane stops at the contract. Test, review, deploy and maintain have docs ([F4](../factory/f4-test.md), [F5](../factory/f5-review.md), [F6](../factory/f6-deploy.md), [F7](../factory/f7-maintain.md)) but no lab of their own yet. For a hands-on run past this lane, use the [cloud sessions](../factory/f11-cloud-sessions.md) lab `advanced-05`, then pick an [assignment](../factory/f10-assignments.md). +Deploy has a doc ([F6](../factory/f6-deploy.md)) but no lab of its own yet. For a hands-on run past this lane, use the [cloud sessions](../factory/f11-cloud-sessions.md) lab `advanced-05`, then pick an [assignment](../factory/f10-assignments.md). diff --git a/labs/factory/04-run-the-contract.md b/labs/factory/04-run-the-contract.md index ae9c377..632d54c 100644 --- a/labs/factory/04-run-the-contract.md +++ b/labs/factory/04-run-the-contract.md @@ -45,7 +45,7 @@ Answer: Every one, because a test that walks them all catches a break in any wor ```sh f=.labs/work/contract.log grep -Eq 'Tests +[0-9]+ passed' $f && ! grep -q 'failed' $f || exit 1 - for t in $(grep -Eo 'tests/contract/[A-Za-z0-9_./-]+[.]test[.]ts' $f); do test -f "$t" && exit 0; done + for t in $(grep -Eo '(tests/contract|factory/engine)/[A-Za-z0-9_./-]+[.]test[.]ts' $f); do test -f "$t" && exit 0; done exit 1 ``` diff --git a/labs/factory/05-run-the-checks.md b/labs/factory/05-run-the-checks.md new file mode 100644 index 0000000..092eed1 --- /dev/null +++ b/labs/factory/05-run-the-checks.md @@ -0,0 +1,62 @@ +# Run the checks that stop a bad test + +**Pairs with:** `docs/factory/f4-test.md` +**Help:** goal + +## Warm-up + +What does the dry run print before a run spends anything? +From: factory-03 +Answer: Each stage with its model, turn cap and budget. + +## Goal + +Run the factory's own check tests and save the whole log to .labs/work/checks.log, then read which checks they cover. These checks (plan-drift, holdout, mutation, manifest-deps) are the evidence the model cannot talk its way past. The proof below must pass. + +## Predict + +A check passes or fails on an exit code or a hash. If the model says "done" but the check fails, which one does the factory believe? + +Answer: The check. The runner reads the exit code, never the model's word. + +## Steps + +1. Run `make lab` and read why it is red. It is red because the work is not done yet; this is the state the steps will change. + Expect: A red result that names the proof items not true yet, then a hint to reveal. Nothing has changed yet. +2. Read the test stage doc. +3. Make the lab work folder. +4. Run every test under `factory/engine/checks` and save the whole log. +5. Read which check files the log lists. + +## Proof + +1. checks.log says passed. + + ```sh + grep -q 'passed' .labs/work/checks.log + ``` +2. checks.log has no failed. + + ```sh + f=.labs/work/checks.log; test -f $f && ! grep -q 'failed' $f + ``` +3. checks.log lists at least three check test files that exist. + + ```sh + f=.labs/work/checks.log + grep -Eq 'Tests +[0-9]+ passed' $f && ! grep -q 'failed' $f || exit 1 + n=0 + for t in $(grep -Eo 'factory/engine/checks/[A-Za-z0-9_./-]+[.]test[.]ts' $f | sort -u); do test -f "$t" && n=$((n+1)); done + test "$n" -ge 3 + ``` + +## Hints + +1. The proof reads `.labs/work/checks.log` for `passed`, no `failed` and the check files by name. +2. Point vitest at the folder `factory/engine/checks`, not at one file. +3. Redirect both output streams into the log with `> file 2>&1`. +4. Commands: `cat docs/factory/f4-test.md`, `mkdir -p .labs/work`, `npx vitest run factory/engine/checks > .labs/work/checks.log 2>&1`, `grep '✓' .labs/work/checks.log`. + +## What you just learned + +Each check is code with its own test. A claim like "the tests pass" is only worth what a check can reproduce from an exit code. diff --git a/labs/factory/06-count-the-votes.md b/labs/factory/06-count-the-votes.md new file mode 100644 index 0000000..d355c75 --- /dev/null +++ b/labs/factory/06-count-the-votes.md @@ -0,0 +1,60 @@ +# Count the votes behind a review + +**Pairs with:** `docs/factory/f5-review.md` +**Help:** goal + +## Warm-up + +Which stage has the largest budget in the dry run? +From: factory-03 +Answer: Build, at three dollars. + +## Goal + +Save the review rules the engine enforces to .labs/work/review-rules.txt and the review tests' log to .labs/work/review.log. The rules file must match what the code says now. The proof below must pass. + +## Predict + +Three verifiers vote on a finding and only two votes are needed. If one verifier says yes and two say no, does the finding reach the pull request? + +Answer: No. It has one vote, below the two needed, so it is dropped. An agent's lone opinion never counts. + +## Steps + +1. Run `make lab` and read why it is red. It is red because the work is not done yet; this is the state the steps will change. + Expect: A red result that names the proof items not true yet, then a hint to reveal. Nothing has changed yet. +2. Read the review stage doc. +3. Make the lab work folder. +4. Print the four review limits from `factory/engine/review.ts` into review-rules.txt. +5. Run the review tests and save the whole log to review.log. + +## Proof + +1. review.log says passed and names review.test.ts. + + ```sh + f=.labs/work/review.log + grep -Eq 'Tests +[0-9]+ passed' $f && ! grep -q 'failed' $f && grep -q 'factory/engine/review.test.ts' $f + ``` +2. review-rules.txt holds the four limits. + + ```sh + grep -Eq '^voters [0-9]+ needed [0-9]+ nits [0-9]+ rounds [0-9]+$' .labs/work/review-rules.txt + ``` +3. review-rules.txt matches the code today. + + ```sh + fresh=$(npx tsx -e 'import("./factory/engine/review.ts").then(m=>console.log("voters",m.VOTERS,"needed",m.NEEDED,"nits",m.MAX_NITS,"rounds",m.MAX_ROUNDS))') + test -n "$fresh" && test "$(cat .labs/work/review-rules.txt)" = "$fresh" + ``` + +## Hints + +1. Two files: review-rules.txt from a one-line script, review.log from vitest. +2. The limits are exported constants in `factory/engine/review.ts`: VOTERS, NEEDED, MAX_NITS and MAX_ROUNDS. +3. The line must read exactly `voters 3 needed 2 nits 3 rounds 3`, so print the constants, do not type them. +4. Commands: `cat docs/factory/f5-review.md`, `mkdir -p .labs/work`, `npx tsx -e 'import("./factory/engine/review.ts").then(m=>console.log("voters",m.VOTERS,"needed",m.NEEDED,"nits",m.MAX_NITS,"rounds",m.MAX_ROUNDS))' > .labs/work/review-rules.txt`, `npx vitest run factory/engine/review.test.ts > .labs/work/review.log 2>&1`. + +## What you just learned + +Review is a vote, not a verdict. Two of three verifiers must agree, nits are capped and the loop stops after three rounds, and the tests pin all four numbers. diff --git a/labs/factory/07-trip-a-band.md b/labs/factory/07-trip-a-band.md new file mode 100644 index 0000000..894719b --- /dev/null +++ b/labs/factory/07-trip-a-band.md @@ -0,0 +1,74 @@ +# Trip a band and read the intent it writes + +**Pairs with:** `docs/factory/f7-maintain.md` +**Help:** goal + +## Warm-up + +Which of the dry run's stages uses Opus? +From: factory-03 +Answer: Delivery. The mechanical stages use Haiku. + +## Goal + +A metric called ci-failure-rate has run between 9 and 11 for ten runs. Pick a new value that is a three sigma breach, save it to .labs/work/latest.txt, and save the intent the factory writes for it to .labs/work/intent.md. No model decides this. The proof below must pass. + +## Predict + +The new run's metric is 12, a little above the usual 10. Does the factory open a new intent for it? + +Answer: No. 12 is within about one sigma of the rolling mean, so at most it logs. Only three sigma writes an intent. + +## Steps + +1. Run `make lab` and read why it is red. It is red because the work is not done yet; this is the state the steps will change. + Expect: A red result that names the proof items not true yet, then a hint to reveal. Nothing has changed yet. +2. Read the maintain stage doc. +3. Make the lab work folder. +4. [editor] Write one number to `.labs/work/latest.txt` that you expect to be a three sigma breach. +5. Run the band check script from hint 4, so it prints the sigma and the action for your number. +6. If the action is not `intent`, pick a bigger number and run it again. Then save the intent it writes. + +## Proof + +1. latest.txt holds one number. + + ```sh + grep -Eq '^[0-9]+([.][0-9]+)?$' .labs/work/latest.txt + ``` +2. That number is a three sigma breach against the fixed history, so the action is intent. + + ```sh + cat > .labs/work/band.ts <<'TS' + import { readFileSync } from "node:fs"; + import { actionFor, detect } from "../../factory/engine/bands.ts"; + const history = [10, 11, 9, 10, 10, 11, 9, 10, 10, 11]; + const latest = Number(readFileSync(".labs/work/latest.txt", "utf8")); + const b = detect("ci-failure-rate", history, latest, "up-is-bad"); + console.log(b.sigma, actionFor(b)); + TS + test "$(npx tsx .labs/work/band.ts)" = "3 intent" + ``` +3. intent.md is the intent the factory writes for your number. + + ```sh + cat > .labs/work/want.ts <<'TS' + import { readFileSync } from "node:fs"; + import { detect, intentFor } from "../../factory/engine/bands.ts"; + const history = [10, 11, 9, 10, 10, 11, 9, 10, 10, 11]; + const latest = Number(readFileSync(".labs/work/latest.txt", "utf8")); + process.stdout.write(intentFor(detect("ci-failure-rate", history, latest, "up-is-bad"))); + TS + test "$(npx tsx .labs/work/want.ts)" = "$(cat .labs/work/intent.md)" + ``` + +## Hints + +1. The history is fixed at 10, 11, 9, 10, 10, 11, 9, 10, 10, 11. Its mean is about 10 and its spread is small. +2. Three sigma is three spreads above the mean. A value only a little above 10 will not do. +3. `intentFor` and `detect` live in `factory/engine/bands.ts`. The proof runs them on your number. +4. Commands: `cat docs/factory/f7-maintain.md`, `mkdir -p .labs/work`, `echo 30 > .labs/work/latest.txt`, then the proof's two scripts as a guide for printing the sigma and the intent into `.labs/work/intent.md`. + +## What you just learned + +A breach is arithmetic, not opinion. One sigma logs, two asks for a diagnosis and three writes the intent.md a new run starts from. diff --git a/tests/structure/lab-artifacts.ts b/tests/structure/lab-artifacts.ts index 85c3d56..cdf8eda 100644 --- a/tests/structure/lab-artifacts.ts +++ b/tests/structure/lab-artifacts.ts @@ -16,6 +16,9 @@ export const NEEDS: Record = { "daily-02": ["api"], "factory-03": [".factory/config.json", "factory/main.ts"], "factory-04": ["factory/engine/workflows.test.ts"], + "factory-05": ["factory/engine/checks"], + "factory-06": ["factory/engine/review.ts"], + "factory-07": ["factory/engine/bands.ts"], "advanced-05": [".claude/settings.json"], }; @@ -89,6 +92,18 @@ export const SOLUTIONS: Record = { `mkdir -p ${W}`, `npx vitest run factory/engine/workflows.test.ts > ${W}/contract.log 2>&1`, ), + "factory-05": sh(`mkdir -p ${W}`, `npx vitest run factory/engine/checks > ${W}/checks.log 2>&1`), + "factory-06": sh( + `mkdir -p ${W}`, + `npx tsx -e 'import("./factory/engine/review.ts").then(m=>console.log("voters",m.VOTERS,"needed",m.NEEDED,"nits",m.MAX_NITS,"rounds",m.MAX_ROUNDS))' > ${W}/review-rules.txt`, + `npx vitest run factory/engine/review.test.ts > ${W}/review.log 2>&1`, + ), + "factory-07": sh( + `mkdir -p ${W}`, + `echo 30 > ${W}/latest.txt`, + `printf 'import{readFileSync as r}from"node:fs";import{detect,intentFor}from"../../factory/engine/bands.ts";process.stdout.write(intentFor(detect("ci-failure-rate",[10,11,9,10,10,11,9,10,10,11],Number(r(".labs/work/latest.txt","utf8")),"up-is-bad")))' > ${W}/mk.ts`, + `npx tsx ${W}/mk.ts > ${W}/intent.md`, + ), "advanced-01": sh( `mkdir -p ${W}/agents`, file( @@ -173,6 +188,13 @@ export const LAZY: Record = { `printf 'stages that would run\\nreview loop\\n' > ${W}/dry-run.txt`, ), "factory-04": sh(`mkdir -p ${W}`, `echo 'Tests 4 passed' > ${W}/contract.log`), + "factory-05": sh(`mkdir -p ${W}`, `echo 'Tests 38 passed' > ${W}/checks.log`), + "factory-06": sh( + `mkdir -p ${W}`, + `echo 'voters 3 needed 2 nits 3 rounds 3' > ${W}/review-rules.txt`, + `echo 'Tests 12 passed' > ${W}/review.log`, + ), + "factory-07": sh(`mkdir -p ${W}`, `echo 10 > ${W}/latest.txt`, `echo x > ${W}/intent.md`), "advanced-01": sh( `mkdir -p ${W}/agents`, `printf -- '---\\nname: reviewer\\ndescription: d\\ntools: Read, Grep, Glob\\nmodel: haiku\\n---\\n' > ${W}/agents/reviewer.md`, From 3def2fc952dc82bc741dcc9fc222e45e65b8f3c1 Mon Sep 17 00:00:00 2001 From: Param Harrison Date: Sun, 1 Nov 2026 15:46:19 +0200 Subject: [PATCH 3/5] Shorten the goal lines of labs 05 and 07 to the 240 character limit Co-Authored-By: Claude Sonnet 5.5 --- labs/factory/05-run-the-checks.md | 2 +- labs/factory/07-trip-a-band.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labs/factory/05-run-the-checks.md b/labs/factory/05-run-the-checks.md index 092eed1..3f04143 100644 --- a/labs/factory/05-run-the-checks.md +++ b/labs/factory/05-run-the-checks.md @@ -11,7 +11,7 @@ Answer: Each stage with its model, turn cap and budget. ## Goal -Run the factory's own check tests and save the whole log to .labs/work/checks.log, then read which checks they cover. These checks (plan-drift, holdout, mutation, manifest-deps) are the evidence the model cannot talk its way past. The proof below must pass. +Run the factory's own check tests and save the whole log to .labs/work/checks.log. These checks (plan-drift, holdout, mutation, manifest-deps) are evidence the model cannot talk past. The proof below must pass. ## Predict diff --git a/labs/factory/07-trip-a-band.md b/labs/factory/07-trip-a-band.md index 894719b..f3dd84b 100644 --- a/labs/factory/07-trip-a-band.md +++ b/labs/factory/07-trip-a-band.md @@ -11,7 +11,7 @@ Answer: Delivery. The mechanical stages use Haiku. ## Goal -A metric called ci-failure-rate has run between 9 and 11 for ten runs. Pick a new value that is a three sigma breach, save it to .labs/work/latest.txt, and save the intent the factory writes for it to .labs/work/intent.md. No model decides this. The proof below must pass. +ci-failure-rate has run between 9 and 11 for ten runs. Save a three sigma breach to .labs/work/latest.txt and the intent the factory writes for it to .labs/work/intent.md. No model decides this. The proof must pass. ## Predict From 2c9b751b787574ba4e290173816f86e334dd7a06 Mon Sep 17 00:00:00 2001 From: Param Harrison Date: Sun, 1 Nov 2026 15:49:51 +0200 Subject: [PATCH 4/5] Give every lab step with a command an Expect line; lab 07 hints carry runnable scripts Co-Authored-By: Claude Sonnet 5.5 --- labs/factory/05-run-the-checks.md | 1 + labs/factory/06-count-the-votes.md | 1 + labs/factory/07-trip-a-band.md | 9 ++++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/labs/factory/05-run-the-checks.md b/labs/factory/05-run-the-checks.md index 3f04143..7fb69ff 100644 --- a/labs/factory/05-run-the-checks.md +++ b/labs/factory/05-run-the-checks.md @@ -26,6 +26,7 @@ Answer: The check. The runner reads the exit code, never the model's word. 2. Read the test stage doc. 3. Make the lab work folder. 4. Run every test under `factory/engine/checks` and save the whole log. + Expect: Nothing prints, because the output goes to the log. It takes a few seconds. 5. Read which check files the log lists. ## Proof diff --git a/labs/factory/06-count-the-votes.md b/labs/factory/06-count-the-votes.md index d355c75..acbc2f2 100644 --- a/labs/factory/06-count-the-votes.md +++ b/labs/factory/06-count-the-votes.md @@ -26,6 +26,7 @@ Answer: No. It has one vote, below the two needed, so it is dropped. An agent's 2. Read the review stage doc. 3. Make the lab work folder. 4. Print the four review limits from `factory/engine/review.ts` into review-rules.txt. + Expect: One line in the file: voters, needed, nits and rounds, each followed by a number. 5. Run the review tests and save the whole log to review.log. ## Proof diff --git a/labs/factory/07-trip-a-band.md b/labs/factory/07-trip-a-band.md index f3dd84b..01753c7 100644 --- a/labs/factory/07-trip-a-band.md +++ b/labs/factory/07-trip-a-band.md @@ -26,8 +26,11 @@ Answer: No. 12 is within about one sigma of the rolling mean, so at most it logs 2. Read the maintain stage doc. 3. Make the lab work folder. 4. [editor] Write one number to `.labs/work/latest.txt` that you expect to be a three sigma breach. -5. Run the band check script from hint 4, so it prints the sigma and the action for your number. -6. If the action is not `intent`, pick a bigger number and run it again. Then save the intent it writes. + Expect: A file with one number and nothing else. +5. Print the sigma and the action for your number with the first script in the hints. + Expect: Two words, such as `3 intent`. A smaller breach prints a smaller sigma. +6. If the action is not `intent`, pick a bigger number and run it again. Then save the intent with the second script in the hints. + Expect: A short markdown file that says the metric is outside its band. ## Proof @@ -67,7 +70,7 @@ Answer: No. 12 is within about one sigma of the rolling mean, so at most it logs 1. The history is fixed at 10, 11, 9, 10, 10, 11, 9, 10, 10, 11. Its mean is about 10 and its spread is small. 2. Three sigma is three spreads above the mean. A value only a little above 10 will not do. 3. `intentFor` and `detect` live in `factory/engine/bands.ts`. The proof runs them on your number. -4. Commands: `cat docs/factory/f7-maintain.md`, `mkdir -p .labs/work`, `echo 30 > .labs/work/latest.txt`, then the proof's two scripts as a guide for printing the sigma and the intent into `.labs/work/intent.md`. +4. Commands: `cat docs/factory/f7-maintain.md`, `mkdir -p .labs/work`, `echo 30 > .labs/work/latest.txt`, `npx tsx -e 'import("./factory/engine/bands.ts").then(m=>{const b=m.detect("ci-failure-rate",[10,11,9,10,10,11,9,10,10,11],Number(process.argv[1]),"up-is-bad");console.log(b.sigma,m.actionFor(b))})' "$(cat .labs/work/latest.txt)"`, `npx tsx -e 'import("./factory/engine/bands.ts").then(m=>process.stdout.write(m.intentFor(m.detect("ci-failure-rate",[10,11,9,10,10,11,9,10,10,11],Number(process.argv[1]),"up-is-bad"))))' "$(cat .labs/work/latest.txt)" > .labs/work/intent.md`. ## What you just learned From 79c80da9e406cdf6e97df23aa7c56b1e54a91f00 Mon Sep 17 00:00:00 2001 From: Param Harrison Date: Sun, 1 Nov 2026 15:52:58 +0200 Subject: [PATCH 5/5] Labs 04 to 06: match 'N passed' so a coloured vitest log passes; lab 07 rejects an empty intent.md Co-Authored-By: Claude Sonnet 5.5 --- labs/factory/04-run-the-contract.md | 2 +- labs/factory/05-run-the-checks.md | 2 +- labs/factory/06-count-the-votes.md | 2 +- labs/factory/07-trip-a-band.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/labs/factory/04-run-the-contract.md b/labs/factory/04-run-the-contract.md index 632d54c..d95c80b 100644 --- a/labs/factory/04-run-the-contract.md +++ b/labs/factory/04-run-the-contract.md @@ -44,7 +44,7 @@ Answer: Every one, because a test that walks them all catches a break in any wor ```sh f=.labs/work/contract.log - grep -Eq 'Tests +[0-9]+ passed' $f && ! grep -q 'failed' $f || exit 1 + grep -Eq '[0-9]+ passed' $f && ! grep -q 'failed' $f || exit 1 for t in $(grep -Eo '(tests/contract|factory/engine)/[A-Za-z0-9_./-]+[.]test[.]ts' $f); do test -f "$t" && exit 0; done exit 1 ``` diff --git a/labs/factory/05-run-the-checks.md b/labs/factory/05-run-the-checks.md index 7fb69ff..9e88906 100644 --- a/labs/factory/05-run-the-checks.md +++ b/labs/factory/05-run-the-checks.md @@ -45,7 +45,7 @@ Answer: The check. The runner reads the exit code, never the model's word. ```sh f=.labs/work/checks.log - grep -Eq 'Tests +[0-9]+ passed' $f && ! grep -q 'failed' $f || exit 1 + grep -Eq '[0-9]+ passed' $f && ! grep -q 'failed' $f || exit 1 n=0 for t in $(grep -Eo 'factory/engine/checks/[A-Za-z0-9_./-]+[.]test[.]ts' $f | sort -u); do test -f "$t" && n=$((n+1)); done test "$n" -ge 3 diff --git a/labs/factory/06-count-the-votes.md b/labs/factory/06-count-the-votes.md index acbc2f2..c77bc62 100644 --- a/labs/factory/06-count-the-votes.md +++ b/labs/factory/06-count-the-votes.md @@ -35,7 +35,7 @@ Answer: No. It has one vote, below the two needed, so it is dropped. An agent's ```sh f=.labs/work/review.log - grep -Eq 'Tests +[0-9]+ passed' $f && ! grep -q 'failed' $f && grep -q 'factory/engine/review.test.ts' $f + grep -Eq '[0-9]+ passed' $f && ! grep -q 'failed' $f && grep -q 'factory/engine/review.test.ts' $f ``` 2. review-rules.txt holds the four limits. diff --git a/labs/factory/07-trip-a-band.md b/labs/factory/07-trip-a-band.md index 01753c7..10d8fb5 100644 --- a/labs/factory/07-trip-a-band.md +++ b/labs/factory/07-trip-a-band.md @@ -62,7 +62,7 @@ Answer: No. 12 is within about one sigma of the rolling mean, so at most it logs const latest = Number(readFileSync(".labs/work/latest.txt", "utf8")); process.stdout.write(intentFor(detect("ci-failure-rate", history, latest, "up-is-bad"))); TS - test "$(npx tsx .labs/work/want.ts)" = "$(cat .labs/work/intent.md)" + test -s .labs/work/intent.md && test "$(npx tsx .labs/work/want.ts)" = "$(cat .labs/work/intent.md)" ``` ## Hints