diff --git a/.github/scripts/post_claude_review.py b/.github/scripts/post_claude_review.py index ba3d212..489f4c3 100755 --- a/.github/scripts/post_claude_review.py +++ b/.github/scripts/post_claude_review.py @@ -81,12 +81,16 @@ # CREDENTIAL scans from every `eyJ`, and a long run of token characters (the first segment of a JWT # holds the letters e, y and J) is rescanned from each one: quadratic on text a diff can steer. -# So a token, with the dots joining its segments, that holds a run longer than MAX_TOKEN_RUN is -# replaced whole before CREDENTIAL looks at the text: no legitimate review text has one, and it -# leaves nothing of a long JWT (its signature included) behind. What is left has runs of at most -# MAX_TOKEN_RUN characters, which CREDENTIAL scans in time linear in the text. +# So a token, with the dots joining its segments, that holds a run longer than MAX_TOKEN_RUN and +# one of the strings every CREDENTIAL alternative starts with (CREDENTIAL_START) is replaced whole +# before CREDENTIAL looks at the text: it leaves nothing of a long JWT (its signature included) +# behind. A long token without one can't match CREDENTIAL, and legitimate review text has them (a +# generated test's file name, a long identifier, a digest), so it is left alone. What is left has +# no run holding a start longer than MAX_TOKEN_RUN characters, which CREDENTIAL scans in time +# linear in the text. MAX_TOKEN_RUN = 128 TOKEN_CHAIN = re.compile(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*") +CREDENTIAL_START = re.compile(r"eyJ|gh[pousr]_|sk-ant-|github_pat_") def redact_credentials(text): @@ -96,7 +100,7 @@ def redact_credentials(text): def long_chain(match): nonlocal count chain = match.group() - if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")): + if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")) and CREDENTIAL_START.search(chain): count += 1 return REDACTED return chain @@ -112,9 +116,11 @@ def long_chain(match): def redact_path(path): """A finding's path with credential-shaped text replaced, and how many were replaced. - Not the long-run rule: a file name is one token, and a long one (a generated test's name, a - hash-named fixture) is no credential, while a path that no longer matches the diff leaves - its finding outside it. A path past MAX_PATH is no real path and gets the full rule. + Not the long-run rule: a long name that holds one of CREDENTIAL_START's strings without being + a credential (a generated test's name with `eyJ` in it) would be replaced whole by it, and a + path that no longer matches the diff leaves its finding outside it. A long name with none of + them is left alone by redact_credentials already. A path past MAX_PATH is no real path and + gets the full rule. """ if len(path) > MAX_PATH: return redact_credentials(path) diff --git a/.github/scripts/tests/test_post_claude_review.py b/.github/scripts/tests/test_post_claude_review.py index 3534f57..920891b 100644 --- a/.github/scripts/tests/test_post_claude_review.py +++ b/.github/scripts/tests/test_post_claude_review.py @@ -1122,7 +1122,7 @@ def test_ordinary_text_with_short_words_is_left_alone(): def test_the_script_and_the_workflow_redact_alike(): rng = random.Random(299) - pieces = [a_jwt(), a_jwt(second=200), GLUED, "ghs_9_" + a_jwt(), "sk-ant-" + "q" * 25, "x-", "eyJ", ".", " ", "/", "a" * 130, "ok"] + pieces = [a_jwt(), a_jwt(second=200), GLUED, "ghs_9_" + a_jwt(), "sk-ant-" + "q" * 25, "x-", "eyJ", ".", " ", "/", "a" * 130, "ok", LONG_NAME] redact_workflow = workflow_credentials()["redact_credentials"] for _ in range(300): text = "".join(rng.choice(pieces) for _ in range(rng.randint(1, 6))) @@ -1168,6 +1168,89 @@ def test_redacting_takes_linear_time_on_long_input(text): LONG_NAME = "tests/Get_Articles_When_" + "A" * 130 + "_Returns_401.cs" +# Long tokens that start no credential: a generated test's name, an identifier, a hex digest, a base64 blob. +LONG_PLAIN_TOKENS = { + "test name": LONG_NAME, + "identifier": "Get" + "Articles" * 30 + "ForTheCurrentUser", + "sha512 digest": "9f86d081884c7d659a2feaa0c55ad015" * 8, + "base64 blob": "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo" * 5 + "==", + "dotted name": "Company." + "Segment" * 30 + ".Service", +} + + +@pytest.mark.parametrize("text", LONG_PLAIN_TOKENS.values(), ids=LONG_PLAIN_TOKENS.keys()) +def test_a_long_token_that_starts_no_credential_is_left_alone_with_no_warning(text): + sentence = f"The finding is in {text}, which is not a secret." + for redact_text in (pcr.redact_credentials, workflow_credentials()["redact_credentials"]): + assert redact_text(sentence) == (sentence, 0) + + +def test_a_long_file_name_quoted_in_a_finding_body_and_the_summary_is_left_alone(): + summary, findings, count = pcr.redact(f"| {LONG_NAME} | added |", [{"path": "a.py", "line": 3, "body": f"See `{LONG_NAME}`."}]) + + assert LONG_NAME in summary + assert LONG_NAME in findings[0]["body"] + assert count == 0 + + +# Each one starts a credential and holds a run over MAX_TOKEN_RUN. CREDENTIAL alone either leaves part of it behind (the ghp, ghu +# and sk-ant cases) or matches none of it (the others), so only the long-run rule replaces it whole, and each start it looks for is +# pinned. +LONG_CREDENTIAL_STARTS = { + "ghp": "ghp_" + "A" * 40 + "_" + "b" * 200, + "ghu": "ghu_" + "A" * 40 + "_" + "b" * 200, + "sk-ant": "sk-ant-" + "a" * 30 + "." + "b" * 200, + "github_pat": "github_pat_" + "a" * 300, + "eyJ in the middle": "prefix" + "q" * 140 + "eyJ" + "q" * 140, + "start in the first segment": "ghp_1." + "b" * 200, + "start in a later segment": "a" * 200 + ".eyJ" + "b" * 20, + "start before the long segment": "sk-ant-" + "a" * 5 + "." + "b" * 200, +} + + +@pytest.mark.parametrize("text", LONG_CREDENTIAL_STARTS.values(), ids=LONG_CREDENTIAL_STARTS.keys()) +def test_a_long_token_that_starts_a_credential_is_still_replaced_whole(text): + for redact_text in (pcr.redact_credentials, workflow_credentials()["redact_credentials"]): + redacted, count = redact_text(f"see {text} end") + + assert redacted == "see [redacted] end" + assert count == 1 + + +def test_a_long_run_in_a_chain_with_no_start_is_left_alone_wherever_it_sits(): + text = "see " + "x" * 200 + ".Service and Company." + "y" * 200 + " end" + for redact_text in (pcr.redact_credentials, workflow_credentials()["redact_credentials"]): + assert redact_text(text) == (text, 0) + + +def test_the_workflow_carries_the_scripts_long_run_rule(): + workflow = workflow_credentials() + + assert workflow["CREDENTIAL_START"].pattern == pcr.CREDENTIAL_START.pattern + assert workflow["MAX_TOKEN_RUN"] == pcr.MAX_TOKEN_RUN + assert workflow["TOKEN_CHAIN"].pattern == pcr.TOKEN_CHAIN.pattern + assert workflow["CREDENTIAL"].pattern == pcr.CREDENTIAL.pattern + + +@pytest.mark.parametrize( + "sample", + [ + "sk-ant-" + "a" * 20, + *(f"gh{letter}_" + "A" * 36 for letter in "pousr"), + "ghs_123_" + a_jwt(), + "github_pat_" + "a" * 22 + "_" + "b" * 59, + a_jwt(), + ], + ids=["sk-ant", "ghp", "gho", "ghu", "ghs", "ghr", "installation token", "github_pat", "jwt"], +) +def test_every_text_credential_matches_starts_with_a_credential_start(sample): + # CREDENTIAL can only match inside a token that CREDENTIAL_START finds, or the long-run rule would skip a run CREDENTIAL scans slowly. + for credential, start in ((pcr.CREDENTIAL, pcr.CREDENTIAL_START), (workflow_credentials()["CREDENTIAL"], workflow_credentials()["CREDENTIAL_START"])): + match = credential.search(sample) + + assert match, sample + assert start.search(match.group()) + def test_a_long_file_name_in_a_finding_path_is_left_alone(): summary, findings, count = pcr.redact("s", [{"path": LONG_NAME, "line": 3, "body": "b"}]) @@ -1176,6 +1259,16 @@ def test_a_long_file_name_in_a_finding_path_is_left_alone(): assert count == 0 +def test_a_long_name_that_holds_a_credential_start_without_being_one_is_left_alone_in_a_finding_path(): + path = "tests/Get_eyJ_" + "A" * 130 + "_Returns_401.cs" + _, findings, count = pcr.redact("s", [{"path": path, "line": 3, "body": "b"}]) + + assert findings[0]["path"] == path + assert count == 0 + # The long-run rule, which a path does not get, would have replaced it whole. + assert pcr.redact_credentials(path) == ("tests/[redacted]", 1) + + def test_a_credential_in_a_finding_path_is_still_redacted(): path = "src/" + a_jwt() + ".cs" _, findings, count = pcr.redact("s", [{"path": path, "line": 3, "body": "b"}]) diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index df55df2..145eb88 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -1742,7 +1742,7 @@ jobs: # can push a change to a PR's scripts but not to this file, which a # pull_request run takes from the PR's merge commit. It also means a PR # cut before this check gets the check its workflow expects. - # post_claude_review.py's CREDENTIAL, MAX_TOKEN_RUN and redact_credentials must match the ones here. + # post_claude_review.py's CREDENTIAL, MAX_TOKEN_RUN, CREDENTIAL_START and redact_credentials must match the ones here. - name: Check Claude's answer for secrets id: answer if: steps.secret.outputs.enabled == 'true' && steps.claude.outputs.structured_output != '' @@ -1773,10 +1773,12 @@ jobs: ) LEAK_WINDOW = 16 - # Mirrors post_claude_review.py: a token holding a run longer than MAX_TOKEN_RUN is replaced - # whole before CREDENTIAL looks, so a long run of `eyJ` isn't rescanned from each one. + # Mirrors post_claude_review.py: a token holding a run longer than MAX_TOKEN_RUN and one of the + # strings CREDENTIAL starts with is replaced whole before CREDENTIAL looks, so a long run of + # `eyJ` isn't rescanned from each one. A long token without one is left alone. MAX_TOKEN_RUN = 128 TOKEN_CHAIN = re.compile(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*") + CREDENTIAL_START = re.compile(r"eyJ|gh[pousr]_|sk-ant-|github_pat_") def redact_credentials(text): @@ -1785,7 +1787,7 @@ jobs: def long_chain(match): nonlocal count chain = match.group() - if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")): + if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")) and CREDENTIAL_START.search(chain): count += 1 return "[redacted]" return chain @@ -1796,7 +1798,7 @@ jobs: def redact_path(path): - # Mirrors post_claude_review.py: a file name is no credential however long it is. + # Mirrors post_claude_review.py: a long name that merely holds a credential start is no credential. if len(path) > 1024: return redact_credentials(path) return CREDENTIAL.subn("[redacted]", path) diff --git a/.sandcastle/lib/agents.mts b/.sandcastle/lib/agents.mts index b50afaa..fce3b1a 100644 --- a/.sandcastle/lib/agents.mts +++ b/.sandcastle/lib/agents.mts @@ -9,17 +9,35 @@ import { readFileSync } from "node:fs"; import * as sandcastle from "@ai-hero/sandcastle"; -import type { Role } from "./config.mts"; +import type { AnyRole, RoleAgent } from "./config.mts"; import { ROLE_AGENTS } from "./repo-config.mts"; import { UncountedStopError } from "./errors.mts"; import { MAX_CAUSE_DEPTH, RunLimits, runLimits, usageLimitLine } from "./limits.mts"; import { usageReport } from "./report.mts"; +// ROLE_AGENTS as every repo may write it: a repo-config.mts written before the +// `ui` role existed names none, and ROLE_AGENTS' own type then has no `ui` key. +const agents: Partial> = ROLE_AGENTS; + +// Whether the repo opted into the UI developer by giving ROLE_AGENTS a `ui` +// entry. A repo that hasn't never plans or runs the role. +export function uiEnabled(): boolean { + return agents.ui !== undefined; +} + +// The agent and budget ROLE_AGENTS gives one role. A role the repo left out +// (only `ui` may be) is a bug in the caller, so it throws. +export function roleAgent(role: AnyRole): RoleAgent { + const agent = agents[role]; + if (agent === undefined) throw new Error(`ROLE_AGENTS in lib/repo-config.mts has no ${role} entry`); + return agent; +} + // The options a role fixes; callers supply everything else. type RoleFixed = "name" | "agent" | "maxIterations" | "signal"; -export function roleOptions(role: Role, timeout: (ms: number) => AbortSignal = AbortSignal.timeout) { - const { model, effort, maxIterations, timeoutMinutes } = ROLE_AGENTS[role]; +export function roleOptions(role: AnyRole, timeout: (ms: number) => AbortSignal = AbortSignal.timeout) { + const { model, effort, maxIterations, timeoutMinutes } = roleAgent(role); return { name: role, agent: sandcastle.claudeCode(model, { effort }), @@ -41,17 +59,17 @@ export function withSharedRules( - role: Role, + role: AnyRole, options: Omit & { output: sandcastle.OutputObjectDefinition }, limits?: RunLimits, ): Promise; export function runRole( - role: Role, + role: AnyRole, options: Omit, limits?: RunLimits, ): Promise; export async function runRole( - role: Role, + role: AnyRole, options: Omit, limits: RunLimits = runLimits, ): Promise { @@ -67,7 +85,7 @@ export async function runRole( export async function runRoleInSandbox( sandbox: sandcastle.Sandbox, - role: Role, + role: AnyRole, options: Omit, limits: RunLimits = runLimits, ): Promise { @@ -113,7 +131,7 @@ export function isRoleTimeout(error: unknown): boolean { // records the stop, so no later role starts or publishes, and rejects with an // UncountedStopError naming the role and the limit's line, with the original // error as its cause. Any other error is rethrown unchanged. -export async function runWithinLimits(role: Role, start: () => Promise, limits: RunLimits = runLimits): Promise { +export async function runWithinLimits(role: AnyRole, start: () => Promise, limits: RunLimits = runLimits): Promise { limits.throwIfStopped(); try { return await start(); diff --git a/.sandcastle/lib/agents.test.mts b/.sandcastle/lib/agents.test.mts index c9e79af..3b23f74 100644 --- a/.sandcastle/lib/agents.test.mts +++ b/.sandcastle/lib/agents.test.mts @@ -2,15 +2,27 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { describe, it } from "node:test"; import type { Sandbox, SandboxRunOptions } from "@ai-hero/sandcastle"; -import { isRoleTimeout, roleOptions, runRoleInSandbox, runWithinLimits, withSharedRules } from "./agents.mts"; +import { isRoleTimeout, roleOptions, uiEnabled, runRoleInSandbox, runWithinLimits, withSharedRules } from "./agents.mts"; import { UncountedStopError } from "./errors.mts"; import { RunLimits } from "./limits.mts"; import { ROLE_AGENTS } from "./repo-config.mts"; import { usageReport } from "./report.mts"; +import { withUiRole } from "./ui-role-fixture.mts"; const sharedRules = readFileSync(new URL("../roles/shared-rules.md", import.meta.url), "utf8"); describe("roleOptions", () => { + it("builds the ui role's options from its ROLE_AGENTS entry, and refuses it when the repo has none", async () => { + await withUiRole(true, () => { + assert.equal(uiEnabled(), true); + assert.equal(roleOptions("ui").name, "ui"); + }); + await withUiRole(false, () => { + assert.equal(uiEnabled(), false); + assert.throws(() => roleOptions("ui"), /no ui entry/); + }); + }); + it("names the run after the role and applies its iteration cap", () => { const options = roleOptions("tester"); diff --git a/.sandcastle/lib/build.mts b/.sandcastle/lib/build.mts index f8c8a66..813758e 100644 --- a/.sandcastle/lib/build.mts +++ b/.sandcastle/lib/build.mts @@ -6,7 +6,7 @@ import { execFileSync } from "node:child_process"; import { existsSync } from "node:fs"; import { setTimeout as sleep } from "node:timers/promises"; import * as sandcastle from "@ai-hero/sandcastle"; -import { isRoleTimeout, runRoleInSandbox } from "./agents.mts"; +import { isRoleTimeout, runRoleInSandbox, uiEnabled } from "./agents.mts"; import { commitsAhead, fetchMain, headOfRef } from "./branches.mts"; import { changedFiles, gateFailureComment, headOf, runCheckpoint, runGate, textBlock, type Checkpoint } from "./checkpoint.mts"; import { @@ -26,7 +26,7 @@ import { commentOnIssue, markerComments, openPullRequest, repoName, type Sandcas import { handBackWorkflowChange, moveToLocalQueue, recordFailedAttempt, type WorkflowChange } from "./handback.mts"; import { repoGitDir, worktreeLinkProblems, worktreePathFor } from "./host-safety.mts"; import { runLimits, type RunLimits } from "./limits.mts"; -import { architectPromptArgs, gateFixerPromptArgs, issuePromptArgs } from "./prompts.mts"; +import { architectPromptArgs, backendPromptArgs, gateFixerPromptArgs, issuePromptArgs } from "./prompts.mts"; import type { Outcome } from "./report.mts"; import { containsSandboxSecret, containsSecret, sandboxSecrets } from "./sandbox-env.mts"; import { describeSecretMatch, firstSecretMatch, type SecretMatch } from "./scan.mts"; @@ -81,7 +81,7 @@ export { UncountedStopError }; // person needs to look at it regardless. // // `docsFailed` is true when the planner picked the scribe for this issue and -// its run failed. Unlike an architect, tester or backend failure, that +// its run failed. Unlike an architect, tester, backend or UI failure, that // doesn't stop the pipeline (see "When a role fails" in // Blazor-Server's docs/plans/sandcastle-workflow.md): the PR still publishes, with a note // that the documentation step failed. @@ -169,7 +169,7 @@ export async function retryOnServerError(step: () => T, wait: (ms: number) => } } -// The issue comment for an architect, tester or backend run that stopped +// The issue comment for an architect, tester, backend or UI run that stopped // the build. `error` is the run's error, already made safe to post (see // publicErrorText): it carries claude-code's stderr or last output lines, so // it's quoted in a text block, which no mention or reference fires in, with @@ -272,14 +272,14 @@ export function latestDesignNote( } // An issue with the optional roles the planner picked for it (see -// resolveRoles in lib/plan.mts): the architect and/or the scribe, each +// resolveRoles in lib/plan.mts): the architect, the UI developer and/or the scribe, each // run only when listed, alongside the tester, developer and reviewer, which always run (see "Phase 6: Build" in // Blazor-Server's docs/plans/sandcastle-workflow.md). export type RoledIssue = SandcastleIssue & { roles: readonly OptionalRole[] }; // The roles that write the design, the tests or the code. When one fails, the // issue stops for the round, unlike the scribe or the reviewer. -type DeveloperRole = "architect" | "tester" | "backend"; +type DeveloperRole = "architect" | "tester" | "backend" | "ui"; // What buildIssue needs from outside the pipeline; tests pass stubs. export type BuildHost = { @@ -691,16 +691,24 @@ async function buildMarkedIssue( const designNotePath = `.sandcastle/work/${issue.number}/design.md`; const prBodyPath = `.sandcastle/work/${issue.number}/pr-body.md`; + // The UI developer runs only for an issue whose plan picked it in a repo that + // opted in. resolveRoles already keeps `ui` out of an opted-out repo's plan; + // this keeps a RoledIssue built another way from running a role the repo has + // no agent or prompt for, after a tester and backend run it can't use. + const uiRuns = issue.roles.includes("ui") && uiEnabled(); + function developerPromptArgs(role: DeveloperRole): sandcastle.PromptArgs { switch (role) { case "architect": return architectPromptArgs(issue, branch, host.latestDesignNote(issue.number)); + case "backend": + return backendPromptArgs(issue, branch, uiRuns); default: return promptArgs; } } - // Run the architect, the tester or the developer (the backend role). + // Run the architect, the tester, the developer (the backend role) or the UI developer. // Returns undefined when the run finished. When it threw, timed out or used // up its iterations without signalling completion, the design, the tests or // the code aren't done, so it returns the build's result and the issue stops @@ -1098,6 +1106,7 @@ async function buildMarkedIssue( ...(issue.roles.includes("architect") ? ["architect" as const] : []), "tester", "backend", + ...(uiRuns ? ["ui" as const] : []), ]; for (const role of developers) { const failed = await developerFails(role); diff --git a/.sandcastle/lib/build.test.mts b/.sandcastle/lib/build.test.mts index 7f5be6f..6f36ec0 100644 --- a/.sandcastle/lib/build.test.mts +++ b/.sandcastle/lib/build.test.mts @@ -39,6 +39,7 @@ import { gitSandboxFake } from "./git-sandbox-fake.mts"; import { RunLimits } from "./limits.mts"; import type { SecretMatch } from "./scan.mts"; import { GATE_COMMAND, GATE_SCRIPT } from "./repo-config.mts"; +import { withUiRole } from "./ui-role-fixture.mts"; // These tests name the Template's gate command, whatever this repo's // GATE_COMMAND is: the fake sandbox reports the configured one as the default. @@ -1730,9 +1731,117 @@ describe("buildIssue publishing a new branch after main changed a workflow", () }); }); +describe("buildIssue ui role", () => { + it("runs the UI developer after the backend developer and before checkpoint 1, when the plan picks it", async () => { + await withUiRole(true, async () => { + const { steps, buildHost } = host([0, 0]); + + await buildIssue(withRoles(["ui"]), branch, base, buildHost); + + assert.deepEqual(steps, [ + "tester", + "backend", + "ui", + "gate: scripts/gate.sh 2>&1", + "reviewer", + "gate: scripts/gate.sh 2>&1", + "publish", + "close", + ]); + }); + }); + + it("runs every optional role in build order when the plan picks them all", async () => { + await withUiRole(true, async () => { + const { steps, buildHost } = host([0, 0]); + + await buildIssue(withRoles(["architect", "ui", "scribe"]), branch, base, buildHost); + + assert.deepEqual(steps.filter((step) => !step.startsWith("gate:")), ["architect", "tester", "backend", "ui", "scribe", "reviewer", "publish", "close"]); + }); + }); + + it("tells the backend developer the UI developer follows, and that it is the last developer run otherwise", async () => { + await withUiRole(true, async () => { + const withUi = host([0, 0]); + await buildIssue(withRoles(["ui"]), branch, base, withUi.buildHost); + const without = host([0, 0]); + await buildIssue(withRoles([]), branch, base, without.buildHost); + + assert.match(String(withUi.runs.find((run) => run.name === "backend")?.promptArgs?.UI_DEVELOPER), /UI developer runs after you/); + assert.match(String(without.runs.find((run) => run.name === "backend")?.promptArgs?.UI_DEVELOPER), /last developer run/); + }); + }); + + it("runs the UI developer from roles/ui.md with the issue's prompt arguments and the shared rules", async () => { + await withUiRole(true, async () => { + const { runs, buildHost } = host([0, 0]); + + await buildIssue(withRoles(["ui"]), branch, base, buildHost); + + const run = runs.find((options) => options.name === "ui")!; + assert.equal(run.promptFile, "./.sandcastle/roles/ui.md"); + assert.equal(run.promptArgs?.TASK_ID, "69"); + assert.equal(run.promptArgs?.BRANCH, branch); + assert.ok(run.promptArgs?.SHARED_RULES); + }); + }); + + it("skips the UI developer, and tells the backend developer it is the last developer run, when the plan names ui in a repo that hasn't opted in", async () => { + await withUiRole(false, async () => { + const { steps, runs, buildHost } = host([0, 0]); + + const result = await buildIssue(withRoles(["ui"]), branch, base, buildHost); + + assert.ok(!steps.includes("ui")); + assert.equal(result.prUrl, "https://github.com/o/r/pull/1"); + assert.match(String(runs.find((run) => run.name === "backend")?.promptArgs?.UI_DEVELOPER), /last developer run/); + }); + }); + + it("publishes nothing and comments on the issue when the UI developer runs out of iterations unfinished", async () => { + await withUiRole(true, async () => { + const { steps, comments, recordBuildFailureCalls, buildHost } = host([0, 0], { unfinished: ["ui"] }); + + const result = await buildIssue(withRoles(["ui"]), branch, base, buildHost); + + assert.ok(!steps.includes("publish")); + assert.equal(result.prUrl, undefined); + assert.equal(recordBuildFailureCalls.length, 1); + assert.match(comments[0]!.body, /ui ran out of iterations/); + }); + }); + + it("stops the build, without publishing, and counts a failed attempt when the UI developer fails", async () => { + await withUiRole(true, async () => { + const { steps, recordBuildFailureCalls, buildHost } = host([0, 0], { failing: ["ui"] }); + + const result = await buildIssue(withRoles(["ui"]), branch, base, buildHost); + + assert.ok(!steps.includes("publish")); + assert.equal(result.prUrl, undefined); + assert.equal(recordBuildFailureCalls.length, 1); + }); + }); + + it("rethrows a usage-limit stop from the UI developer instead of counting a failed attempt", async () => { + await withUiRole(true, async () => { + const stop = new UncountedStopError("usage limit reached"); + const { recordBuildFailureCalls, buildHost } = host([0, 0], { failing: ["ui"], failWith: { ui: stop } }); + + await assert.rejects( + () => buildIssue(withRoles(["ui"]), branch, base, buildHost), + (error: unknown) => error === stop, + ); + + assert.deepEqual(recordBuildFailureCalls, []); + }); + }); +}); + // Covers Blazor-Server#72 ("Add architect, UI developer and scribe roles -// picked by the planner"; the Template has no UI developer): the build order is -// [architect] → tester → backend → gate 1 → [scribe] → reviewer → gate 2 → publish, +// picked by the planner"): the build order is +// [architect] → tester → backend → [ui] → gate 1 → [scribe] → reviewer → gate 2 → publish, // with each bracketed role run only when the planner picked it (issue.roles). describe("buildIssue optional roles", () => { it("runs tester, backend, scribe and reviewer, and no architect, when the plan picks only the scribe", async () => { @@ -2158,7 +2267,7 @@ describe("buildIssue marking the issue as building", () => { // (see config.test.mts) must outlast. const { runs, buildHost } = host([1, 1, 0, 1, 1, 0]); - const result = await buildIssue(withRoles(["architect", "scribe"]), branch, base, buildHost); + const result = await withUiRole(true, () => buildIssue(withRoles(["architect", "ui", "scribe"]), branch, base, buildHost)); const roles = runs.map((run) => run.name!); const others = roles.filter((role) => role !== "gate-fixer"); diff --git a/.sandcastle/lib/checkpoint.test.mts b/.sandcastle/lib/checkpoint.test.mts index 49e6ee7..67f5713 100644 --- a/.sandcastle/lib/checkpoint.test.mts +++ b/.sandcastle/lib/checkpoint.test.mts @@ -344,7 +344,7 @@ describe("the gate guard in a git repository", () => { git("commit", "--quiet", "-m", "base"); await test({ git, write, run }); } finally { - rmSync(dir, { recursive: true, force: true }); + rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }; @@ -487,7 +487,7 @@ describe("changedFiles", () => { assert.deepEqual(await changedFiles(run, { base }, ["scripts/"]), ["scripts/a` closes #12 `b.sh", "scripts/gone.sh", "scripts/keep.sh"]); assert.equal(await changedFiles(run, { base: "0".repeat(40) }, ["scripts/"]), undefined); } finally { - rmSync(dir, { recursive: true, force: true }); + rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); diff --git a/.sandcastle/lib/config.mts b/.sandcastle/lib/config.mts index 5bb0d48..de8f01a 100644 --- a/.sandcastle/lib/config.mts +++ b/.sandcastle/lib/config.mts @@ -9,11 +9,23 @@ import type { ClaudeCodeOptions } from "@ai-hero/sandcastle"; // Every role Sandcastle runs. lib/repo-config.mts gives each one its agent // and budget (ROLE_AGENTS) and must name every one: a role it leaves out, -// or one it adds, fails the type-check rather than a run. +// or one it adds besides the opt-in `ui` (OptInRole), fails the type-check +// rather than a run. export const ROLES = ["intake", "planner", "critique", "architect", "tester", "backend", "scribe", "reviewer", "gate-fixer", "follow-up"] as const; export type Role = (typeof ROLES)[number]; +// The role a repo opts into by giving ROLE_AGENTS a `ui` entry: the UI +// developer, who builds the Blazor components and pages after the backend +// developer. It is left out of ROLES so that a repo-config.mts written before +// it existed, whose ROLE_AGENTS must name every Role, still type-checks. +// lib/agents.mts#uiEnabled says whether the repo opted in, and nothing runs +// or plans the role until it has. +export type OptInRole = "ui"; + +// Every role the Owned modules can run: the ones every repo names, plus the opt-in one. +export type AnyRole = Role | OptInRole; + // The agent and budget lib/repo-config.mts#ROLE_AGENTS gives one role. export type RoleAgent = { model: string; @@ -26,8 +38,10 @@ export type RoleAgent = { // (the backend role) and reviewer, which always run (see "Phase 4: Plan" and // "Phase 6: Build" in Blazor-Server's docs/plans/sandcastle-workflow.md). lib/plan.mts#resolveRoles // validates the planner's roles field against this list, falling back to -// every one of them when the field is missing or invalid. -export const OPTIONAL_ROLES = ["architect", "scribe"] as const satisfies readonly Role[]; +// every one of them when the field is missing or invalid. `ui` counts only in a +// repo that opted in (lib/agents.mts#uiEnabled): elsewhere it is neither +// recognised nor part of the fallback. +export const OPTIONAL_ROLES = ["architect", "ui", "scribe"] as const satisfies readonly AnyRole[]; export type OptionalRole = (typeof OPTIONAL_ROLES)[number]; @@ -41,7 +55,7 @@ export const DESIGN_MARKER = ""; // checkpoint. BUILDING_LABEL_MAX_AGE_MS is sized from these roles' timeouts: // config.test.mts checks the sum fits, and build.test.mts that buildIssue runs // no role outside the list. -export const BUILD_ROLES = ["architect", "tester", "backend", "scribe", "reviewer"] as const satisfies readonly Role[]; +export const BUILD_ROLES = ["architect", "tester", "backend", "ui", "scribe", "reviewer"] as const satisfies readonly AnyRole[]; // The ref each issue branch is compared with: every issue PR targets main, // and fetchMain() refreshes origin/main before each round. The sandbox mounts diff --git a/.sandcastle/lib/config.test.mts b/.sandcastle/lib/config.test.mts index 6c4293e..51bc42a 100644 --- a/.sandcastle/lib/config.test.mts +++ b/.sandcastle/lib/config.test.mts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; import { describe, it } from "node:test"; import { BUILD_ROLES, @@ -13,17 +14,30 @@ import { budgetMinutesFrom, queueScopeFrom, } from "./config.mts"; +import { roleAgent, uiEnabled } from "./agents.mts"; import { ROLE_AGENTS } from "./repo-config.mts"; describe("ROLE_AGENTS", () => { // lib/repo-config.mts gives each role its agent. The Owned modules index it by Role, so a role it lacks fails a run, and one - // it adds that ROLES lacks fails the type-check. - it("gives every role in ROLES an agent, and no other", () => { - assert.deepEqual(Object.keys(ROLE_AGENTS).sort(), [...ROLES].sort()); + // it adds that ROLES lacks fails the type-check, except the opt-in `ui`, which is not in ROLES so that a + // repo-config.mts written before it existed still type-checks. + it("gives every role in ROLES an agent, and no other but the opt-in ui", () => { + assert.deepEqual(Object.keys(ROLE_AGENTS).filter((role) => role !== "ui").sort(), [...ROLES].sort()); }); it("names every role the build and the optional-role list use", () => { - for (const role of [...BUILD_ROLES, ...OPTIONAL_ROLES]) assert.ok(ROLES.includes(role), `${role} is missing from ROLES`); + for (const role of [...BUILD_ROLES, ...OPTIONAL_ROLES]) { + assert.ok(role === "ui" || ROLES.includes(role), `${role} is missing from ROLES`); + } + }); + + // A repo that adds a ui entry also needs the prompts the role and its planner pick read, or its first ui build fails. + it("has the prompts the ui role needs when the repo opted into it", { skip: !uiEnabled() && "the repo hasn't opted into the ui role" }, () => { + const file = (path: string) => new URL(`../${path}`, import.meta.url); + + assert.ok(existsSync(file("roles/ui.md")), "roles/ui.md is missing"); + assert.match(readFileSync(file("roles/backend.md"), "utf8"), /\{\{UI_DEVELOPER\}\}/); + assert.match(readFileSync(file("plan-prompt.md"), "utf8"), /^- `ui`:/m); }); }); @@ -35,8 +49,11 @@ describe("BUILDING_LABEL_MAX_AGE_MS", () => { // checkpoints. build.test.mts checks buildIssue runs no other role. it("outlasts the longest build the role timeouts allow, with an hour for gates, sandboxes and publishing", () => { const checkpoints = 2; + // The UI developer counts even in a repo that hasn't opted in, at the backend developer's budget (the Seed's example + // entry), so a repo that opts in with a larger one gets the sum checked again by this test in its own config. + const timeout = (role: (typeof BUILD_ROLES)[number]) => (role === "ui" && !uiEnabled() ? ROLE_AGENTS.backend : roleAgent(role)).timeoutMinutes; const minutes = - BUILD_ROLES.reduce((sum, role) => sum + ROLE_AGENTS[role].timeoutMinutes, 0) + + BUILD_ROLES.reduce((sum, role) => sum + timeout(role), 0) + checkpoints * GATE_FIXER_ATTEMPTS * ROLE_AGENTS["gate-fixer"].timeoutMinutes; const hour = 60; diff --git a/.sandcastle/lib/git-sandbox-fake.mts b/.sandcastle/lib/git-sandbox-fake.mts index 4e49e5c..afac241 100644 --- a/.sandcastle/lib/git-sandbox-fake.mts +++ b/.sandcastle/lib/git-sandbox-fake.mts @@ -103,6 +103,6 @@ export function gitSandboxFake(): GitSandboxFake { main, globalConfig: () => readFileSync(globalConfigPath, "utf8"), headIdentity: () => run("git log -1 --format='%an <%ae>|%cn <%ce>'"), - cleanup: () => rmSync(root, { recursive: true, force: true }), + cleanup: () => rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }), }; } diff --git a/.sandcastle/lib/host-safety.test.mts b/.sandcastle/lib/host-safety.test.mts index 8629be4..d6b7f84 100644 --- a/.sandcastle/lib/host-safety.test.mts +++ b/.sandcastle/lib/host-safety.test.mts @@ -15,7 +15,7 @@ describe("worktreeConfigProblem", () => { setup(join(root, "repo", ".git")); return worktreeConfigProblem(join(root, "repo", ".git")); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }; const setExtension = (value: string) => (gitDir: string) => @@ -44,7 +44,7 @@ describe("configIncludeProblem", () => { setup(join(gitDir, "config"), join(root, "outside")); return configIncludeProblem(gitDir); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }; const set = (key: string, value: string) => (config: string) => execFileSync("git", ["config", "--file", config, key, value]); @@ -97,7 +97,7 @@ describe("protectHostGit", () => { encoding: "utf8", }); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }; @@ -155,7 +155,7 @@ describe("withoutGitHooks", () => { git(process.env, "commit", "--quiet", "--allow-empty", "-m", "unguarded"); assert.equal(existsSync(ran), true); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); @@ -200,7 +200,7 @@ describe("protectedGitMounts", () => { assert.equal(existsSync(join(gitDir, "review")), true); } finally { - rmSync(gitDir, { recursive: true, force: true }); + rmSync(gitDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -212,7 +212,7 @@ describe("protectedGitMounts", () => { assert.throws(() => protectedGitMounts(gitDir, existsSync, "/empty"), (error: Error) => error.message.startsWith(`${join(gitDir, "review")} isn't a real folder`)); } finally { - rmSync(gitDir, { recursive: true, force: true }); + rmSync(gitDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -227,7 +227,7 @@ describe("protectedGitMounts", () => { assert.throws(() => protectedGitMounts(gitDir, existsSync, "/empty"), (error: Error) => error.message.startsWith(`${join(gitDir, "review")} isn't a real folder`)); } finally { - rmSync(gitDir, { recursive: true, force: true }); + rmSync(gitDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); } @@ -243,7 +243,7 @@ describe("protectedGitMounts", () => { assert.equal(statSync(join(gitDir, "review")).mode & 0o777, 0o700); } finally { - rmSync(gitDir, { recursive: true, force: true }); + rmSync(gitDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -274,7 +274,7 @@ describe("worktreeLinkProblems", () => { try { assert.deepEqual(worktreeLinkProblems(worktree, commonDir), []); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -285,7 +285,7 @@ describe("worktreeLinkProblems", () => { writeFileSync(join(worktree, ".git"), `gitdir: ${join(worktree, "evil")}\n`); assert.match(worktreeLinkProblems(worktree, commonDir).join(), /doesn't point into/); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -295,7 +295,7 @@ describe("worktreeLinkProblems", () => { writeFileSync(join(commonDir, "worktrees", "feature-1-x", "commondir"), `${join(root, "evil")}\n`); assert.match(worktreeLinkProblems(worktree, commonDir).join(), /doesn't lead back/); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -307,7 +307,7 @@ describe("worktreeLinkProblems", () => { symlinkSync(`${admin}-moved`, admin); assert.match(worktreeLinkProblems(worktree, commonDir).join(), /symlink/); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); @@ -346,7 +346,7 @@ describe("hostGitEnv", () => { } assert.equal(existsSync(ran), true); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); @@ -358,7 +358,7 @@ describe("mainGitDir", () => { execFileSync("git", ["init", "--quiet"], { cwd: root, stdio: "ignore" }); assert.equal(mainGitDir(root), join(root, ".git")); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -369,7 +369,7 @@ describe("mainGitDir", () => { writeFileSync(join(root, ".git", "commondir"), "evil\n"); assert.throws(() => mainGitDir(root), /commondir exists/); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -378,7 +378,7 @@ describe("mainGitDir", () => { try { assert.throws(() => mainGitDir(root), /isn't a directory/); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); diff --git a/.sandcastle/lib/plan.mts b/.sandcastle/lib/plan.mts index 11796c9..4ce28e6 100644 --- a/.sandcastle/lib/plan.mts +++ b/.sandcastle/lib/plan.mts @@ -1,6 +1,6 @@ import * as sandcastle from "@ai-hero/sandcastle"; import { z } from "zod"; -import { runRole } from "./agents.mts"; +import { runRole, uiEnabled } from "./agents.mts"; import { OPTIONAL_ROLES, type OptionalRole } from "./config.mts"; import { hooks } from "./repo-config.mts"; import type { SandcastleIssue } from "./github.mts"; @@ -26,18 +26,21 @@ const planSchema = z.object({ export type PlannedIssue = z.infer["issues"][number]; -// The optional roles (architect, scribe) resolveRoles picks for +// The optional roles (architect, UI developer, scribe) resolveRoles picks for // one issue from the planner's raw `roles` field: the recognised ones, // deduplicated and in build order, or every optional role when the field is // missing, isn't an array, or names anything resolveRoles doesn't recognise // (see OPTIONAL_ROLES in lib/config.mts). An empty array is valid on its own: -// it means the planner deliberately picked none. +// it means the planner deliberately picked none. The `ui` role counts only in +// a repo that opted into it (see uiEnabled in lib/agents.mts): elsewhere it is +// neither recognised nor part of "every optional role". export function resolveRoles(roles: unknown): OptionalRole[] { - const known: readonly string[] = OPTIONAL_ROLES; + const available = OPTIONAL_ROLES.filter((role) => role !== "ui" || uiEnabled()); + const known: readonly string[] = available; if (!Array.isArray(roles) || !roles.every((role) => typeof role === "string" && known.includes(role))) { - return [...OPTIONAL_ROLES]; + return [...available]; } - return OPTIONAL_ROLES.filter((role) => roles.includes(role)); + return available.filter((role) => roles.includes(role)); } // Where the repository is checked out inside the sandbox the planner runs in. diff --git a/.sandcastle/lib/plan.test.mts b/.sandcastle/lib/plan.test.mts index 83fd2df..65b6dfa 100644 --- a/.sandcastle/lib/plan.test.mts +++ b/.sandcastle/lib/plan.test.mts @@ -1,7 +1,9 @@ import assert from "node:assert/strict"; import { describe, it } from "node:test"; import { OPTIONAL_ROLES } from "./config.mts"; +import { uiEnabled } from "./agents.mts"; import { readyPicks, resolveFiles, resolveRoles } from "./plan.mts"; +import { withUiRole } from "./ui-role-fixture.mts"; describe("readyPicks", () => { const ready = [{ number: 4 }, { number: 5 }]; @@ -47,19 +49,22 @@ describe("readyPicks", () => { // fallback, since the plan schema itself can't reject an invalid field // without aborting the whole plan (see plan.mts). describe("resolveRoles", () => { + // The roles a plan without a valid `roles` field runs: the optional ones, `ui` only when the repo opted in. + const everyOptionalRole = () => OPTIONAL_ROLES.filter((role) => role !== "ui" || uiEnabled()); + it("runs every optional role when the field is missing", () => { - assert.deepEqual(resolveRoles(undefined), [...OPTIONAL_ROLES]); + assert.deepEqual(resolveRoles(undefined), everyOptionalRole()); }); it("runs every optional role when the field isn't an array", () => { - assert.deepEqual(resolveRoles("architect"), [...OPTIONAL_ROLES]); - assert.deepEqual(resolveRoles(null), [...OPTIONAL_ROLES]); - assert.deepEqual(resolveRoles({ architect: true }), [...OPTIONAL_ROLES]); + assert.deepEqual(resolveRoles("architect"), everyOptionalRole()); + assert.deepEqual(resolveRoles(null), everyOptionalRole()); + assert.deepEqual(resolveRoles({ architect: true }), everyOptionalRole()); }); it("runs every optional role when the array names something resolveRoles doesn't recognise", () => { - assert.deepEqual(resolveRoles(["architect", "backend"]), [...OPTIONAL_ROLES]); - assert.deepEqual(resolveRoles(["frontend"]), [...OPTIONAL_ROLES]); + assert.deepEqual(resolveRoles(["architect", "backend"]), everyOptionalRole()); + assert.deepEqual(resolveRoles(["frontend"]), everyOptionalRole()); }); it("keeps only the roles the planner picked, in build order", () => { @@ -71,6 +76,22 @@ describe("resolveRoles", () => { assert.deepEqual(resolveRoles([]), []); }); + it("leaves the ui role out of every optional role, and refuses a plan that picks it, in a repo that hasn't opted in", async () => { + await withUiRole(false, () => { + const everyOtherRole = OPTIONAL_ROLES.filter((role) => role !== "ui"); + assert.deepEqual(resolveRoles(undefined), everyOtherRole); + assert.deepEqual(resolveRoles(["ui"]), everyOtherRole); + assert.deepEqual(resolveRoles(["architect"]), ["architect"]); + }); + }); + + it("includes the ui role, in build order, in a repo that opted in", async () => { + await withUiRole(true, () => { + assert.deepEqual(resolveRoles(undefined), ["architect", "ui", "scribe"]); + assert.deepEqual(resolveRoles(["scribe", "ui", "architect"]), ["architect", "ui", "scribe"]); + }); + }); + it("drops a repeated role rather than running it twice", () => { assert.deepEqual(resolveRoles(["scribe", "scribe"]), ["scribe"]); }); diff --git a/.sandcastle/lib/prompts.mts b/.sandcastle/lib/prompts.mts index 79d1557..e7537bf 100644 --- a/.sandcastle/lib/prompts.mts +++ b/.sandcastle/lib/prompts.mts @@ -66,6 +66,20 @@ export function architectPromptArgs(issue: SandcastleIssue, branch: string, desi return { ...issuePromptArgs(issue, branch), DESIGN_NOTE: designNote ?? "(no earlier design note)" }; } +// The backend developer needs to know whether the UI developer runs after it +// for this issue: when it does, the Blazor components and pages are the UI +// developer's, and the backend's run needn't leave every test green. +export function backendPromptArgs(issue: SandcastleIssue, branch: string, uiRuns: boolean): PromptArgs { + return { + ...issuePromptArgs(issue, branch), + UI_DEVELOPER: uiRuns + ? "The UI developer runs after you for this issue. Leave Blazor components and pages (`.razor` files, their code-behind and their " + + "bUnit tests) to it, and make every other test pass. If the gate is still red only on those tests when you finish, say so in your " + + "last commit's body." + : "No UI developer runs for this issue, so your run is the last developer run: every test must pass by its end.", + }; +} + // The host names branches and has already dropped issues with an open PR, so // the planner needs only the ready issues. export function plannerPromptArgs(ready: SandcastleIssue[]) { diff --git a/.sandcastle/lib/review-ledger.test.mts b/.sandcastle/lib/review-ledger.test.mts index b7b5cb5..81dba8a 100644 --- a/.sandcastle/lib/review-ledger.test.mts +++ b/.sandcastle/lib/review-ledger.test.mts @@ -139,7 +139,7 @@ beforeEach(() => { }); afterEach(() => { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); }); describe("diffKey", () => { @@ -498,7 +498,7 @@ describe("diffKey", () => { assert.doesNotThrow(() => diffKey(repo, mergeBase(repo))); } finally { process.env.PATH = path; - rmSync(bin, { recursive: true, force: true }); + rmSync(bin, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -732,7 +732,7 @@ describe("checkReview", () => { return checkReview(repo, mergeBase(repo), {}); } finally { process.env.PATH = path; - rmSync(bin, { recursive: true, force: true }); + rmSync(bin, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } } diff --git a/.sandcastle/lib/scan.test.mts b/.sandcastle/lib/scan.test.mts index cdeedd9..f8fbe00 100644 --- a/.sandcastle/lib/scan.test.mts +++ b/.sandcastle/lib/scan.test.mts @@ -96,7 +96,7 @@ describe("firstSecretMatch", () => { try { assert.equal(firstSecretMatch(base, leak, new Map([["ANTHROPIC_API_KEY", "unrelated-secret-value"]]), hostGit), undefined); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -114,7 +114,7 @@ describe("firstSecretMatch", () => { // Never the matched text itself. assert.ok(!JSON.stringify(match).includes(githubShaped)); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -128,7 +128,7 @@ describe("firstSecretMatch", () => { // Never the matched text itself. assert.ok(!JSON.stringify(match).includes(secretValue)); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -143,7 +143,7 @@ describe("firstSecretMatch", () => { // Never the matched text itself. assert.ok(!JSON.stringify(match).includes(secretValue)); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); } @@ -158,7 +158,7 @@ describe("firstSecretMatch", () => { // Never the matched text itself. assert.ok(!JSON.stringify(match).includes(secretValue)); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -169,7 +169,7 @@ describe("firstSecretMatch", () => { try { assert.deepEqual(firstSecretMatch(base, leak, new Map(), hostGit)?.location, { commit: leak, field: "line", path: fileName, line: 1 }); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); } @@ -184,7 +184,7 @@ describe("firstSecretMatch", () => { assert.deepEqual(firstSecretMatch(base, second, new Map(), hostGit)?.location, { commit: second, field: "line", path: "notes.txt", line: 4 }); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -199,7 +199,7 @@ describe("firstSecretMatch", () => { assert.equal(firstSecretMatch(base, leak, new Map([["ANTHROPIC_API_KEY", secretValue]]), hostGit)?.location.field, "message"); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -215,7 +215,7 @@ describe("firstSecretMatch", () => { assert.equal(firstSecretMatch(base, leak, new Map([["ANTHROPIC_API_KEY", secretValue]]), hostGit)?.location.field, "message"); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); @@ -260,7 +260,7 @@ describe("describeSecretMatch", () => { assert.ok(!text.includes(matched.slice(-8)), text); } } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); @@ -297,7 +297,7 @@ describe("firstSecretMatch with a merged main commit", () => { try { assert.deepEqual(firstSecretMatch(base, merged, new Map(), hostGit)?.location, { commit: main, field: "line", path: "main.txt", line: 1 }); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -306,7 +306,7 @@ describe("firstSecretMatch with a merged main commit", () => { try { assert.equal(firstSecretMatch(base, merged, new Map(), hostGit, main), undefined); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -318,7 +318,7 @@ describe("firstSecretMatch with a merged main commit", () => { assert.equal(match?.location.field, "line"); assert.equal(match?.location.field === "line" ? match.location.path : undefined, "branch.txt"); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -334,7 +334,7 @@ describe("firstSecretMatch with a merged main commit", () => { assert.deepEqual(match?.location, { commit: mergeCommit, field: "line", path: "resolved.txt", line: 1 }); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); @@ -349,7 +349,7 @@ describe("firstSecretMatch with a merged main commit", () => { assert.deepEqual(match, { location: { commit: tip, field: "message" }, kind: { type: "secret", envKey: "ANTHROPIC_API_KEY" } }); } finally { - rmSync(repo, { recursive: true, force: true }); + rmSync(repo, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); diff --git a/.sandcastle/lib/shell.test.mts b/.sandcastle/lib/shell.test.mts index 00130f7..b5d25e3 100644 --- a/.sandcastle/lib/shell.test.mts +++ b/.sandcastle/lib/shell.test.mts @@ -46,7 +46,7 @@ describe("hostGitInvocation", () => { assert.equal(existsSync(ran), false, "the pre-push hook ran on the host"); assert.equal(run(root, ["--git-dir", remote, "rev-parse", "refs/heads/fix/1-x"]), commit); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); @@ -60,7 +60,7 @@ describe("runHostGit", () => { /git rev-parse HEAD failed:\n.*(not a git repository|fatal)/s, ); } finally { - rmSync(dir, { recursive: true, force: true }); + rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); }); diff --git a/.sandcastle/lib/skills.test.mts b/.sandcastle/lib/skills.test.mts index 965fb51..b024550 100644 --- a/.sandcastle/lib/skills.test.mts +++ b/.sandcastle/lib/skills.test.mts @@ -1,8 +1,10 @@ import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, it } from "node:test"; +import { uiEnabled } from "./agents.mts"; +import { SANDBOX_SKILLS } from "./repo-config.mts"; import { makeAgentSandbox, skillMounts, withCreateMounts, type SkillMount } from "./skills.mts"; function skill(dir: string): string { @@ -259,3 +261,71 @@ describe("makeAgentSandbox", () => { assert.deepEqual(stub.logs, []); }); }); + +// The skills a role prompt's "# SKILLS" section lists, as `- \`name\`` bullets (a `: description` after the name is optional). Line +// endings may be CRLF, the heading may carry trailing spaces, and lines inside a code fence are neither headings nor bullets. +function listedSkills(prompt: string): string[] { + const listed: string[] = []; + let inSection = false; + let inFence = false; + for (const line of prompt.split(/\r?\n/)) { + if (/^\s*(```|~~~)/.test(line)) inFence = !inFence; + else if (inFence) continue; + else if (/^# /.test(line)) inSection = /^# SKILLS\s*$/.test(line); + else if (inSection) { + const name = /^- `([^`]+)`/.exec(line)?.[1]; + if (name !== undefined) listed.push(name); + } + } + return listed; +} + +describe("listedSkills", () => { + it("reads the bullets of the SKILLS section only", () => { + const prompt = "# TASK\n\n- `not-a-skill`: x\n\n# SKILLS\n\nIntro.\n\n- `dotnet-tdd`: a\n- `dotnet-xunit`: b\n\n# EXECUTION\n\n- `also-not`: y\n"; + + assert.deepEqual(listedSkills(prompt), ["dotnet-tdd", "dotnet-xunit"]); + }); + + it("reads a prompt with CRLF line endings, a heading with trailing spaces and a bullet without a colon", () => { + const prompt = "# TASK\r\n\r\n# SKILLS \r\n\r\n- `dotnet-tdd`: a\r\n- `dotnet-xunit`\r\n\r\n# EXECUTION\r\n"; + + assert.deepEqual(listedSkills(prompt), ["dotnet-tdd", "dotnet-xunit"]); + }); + + it("ignores a heading or bullet inside a code fence", () => { + const prompt = "# SKILLS\n\n- `dotnet-tdd`: a\n\n```markdown\n# EXECUTION\n- `in-fence`: b\n```\n\n- `dotnet-xunit`: c\n"; + + assert.deepEqual(listedSkills(prompt), ["dotnet-tdd", "dotnet-xunit"]); + }); + + it("finds none in a prompt without the section", () => { + assert.deepEqual(listedSkills("# TASK\n\n- `dotnet-tdd`: a\n"), []); + }); +}); + +// SANDBOX_SKILLS mounts the skills; the prompts tell the agent to use them. The two must agree, or a prompt promises a skill the +// sandbox doesn't have, or the sandbox mounts one no prompt mentions. +describe("role prompts and SANDBOX_SKILLS", () => { + const rolesDir = new URL("../roles/", import.meta.url); + const prompt = (role: string) => readFileSync(new URL(`${role}.md`, rolesDir), "utf8"); + const roles = readdirSync(rolesDir).filter((file) => file.endsWith(".md")).map((file) => file.slice(0, -3)); + + it("lists no skill that SANDBOX_SKILLS doesn't mount, in any role prompt", () => { + for (const role of roles) { + const unmounted = listedSkills(prompt(role)).filter((name) => !SANDBOX_SKILLS.includes(name)); + + assert.deepEqual(unmounted, [], `roles/${role}.md lists skills lib/repo-config.mts doesn't mount`); + } + }); + + // The roles that write code and tests each list every mounted skill; the ui role only in a repo that opted into it. + for (const role of ["tester", "backend", "ui"]) { + it(`lists every mounted skill in the ${role} prompt`, { skip: role === "ui" && !uiEnabled() && "the repo hasn't opted into the ui role" }, () => { + const listed = listedSkills(prompt(role)); + const missing = SANDBOX_SKILLS.filter((name) => !listed.includes(name)); + + assert.deepEqual(missing, [], `roles/${role}.md doesn't list skills lib/repo-config.mts mounts`); + }); + } +}); diff --git a/.sandcastle/lib/ui-role-fixture.mts b/.sandcastle/lib/ui-role-fixture.mts new file mode 100644 index 0000000..e4144b6 --- /dev/null +++ b/.sandcastle/lib/ui-role-fixture.mts @@ -0,0 +1,20 @@ +// Test support for the opt-in ui role (see OptInRole in lib/config.mts): runs a function with the +// repo's `ui` entry in ROLE_AGENTS present or absent, whatever lib/repo-config.mts says, so a test of +// the role doesn't depend on the repo having opted in or out. + +import { ROLE_AGENTS } from "./repo-config.mts"; + +// Runs `run` with ROLE_AGENTS holding a `ui` entry (a copy of the backend's) or none, then puts +// ROLE_AGENTS back as it was, on every path. +export async function withUiRole(enabled: boolean, run: () => T | Promise): Promise { + const agents: Record = ROLE_AGENTS; + const original = agents.ui; + if (enabled) agents.ui = { ...ROLE_AGENTS.backend }; + else delete agents.ui; + try { + return await run(); + } finally { + if (original === undefined) delete agents.ui; + else agents.ui = original; + } +} diff --git a/.sandcastle/lib/ui-role-fixture.test.mts b/.sandcastle/lib/ui-role-fixture.test.mts new file mode 100644 index 0000000..7d03129 --- /dev/null +++ b/.sandcastle/lib/ui-role-fixture.test.mts @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { ROLE_AGENTS } from "./repo-config.mts"; +import { withUiRole } from "./ui-role-fixture.mts"; + +const agents: Record = ROLE_AGENTS; + +describe("withUiRole", () => { + it("gives ROLE_AGENTS a ui entry that copies the backend's, not the backend's own object", async () => { + await withUiRole(true, () => { + assert.deepEqual(agents.ui, ROLE_AGENTS.backend); + assert.notEqual(agents.ui, ROLE_AGENTS.backend); + }); + }); + + it("leaves ROLE_AGENTS without a ui entry when asked to", async () => { + await withUiRole(false, () => { + assert.equal("ui" in agents, false); + }); + }); + + it("returns what the callback returns, whether it is sync or async", async () => { + assert.equal(await withUiRole(true, () => 7), 7); + assert.equal(await withUiRole(false, async () => "x"), "x"); + }); + + it("puts ROLE_AGENTS back as it found it, opted in or out, even when the callback throws", async () => { + const before = agents.ui; + for (const enabled of [true, false]) { + await assert.rejects( + withUiRole(enabled, () => { + throw new Error("boom"); + }), + /boom/, + ); + + assert.equal(agents.ui, before); + assert.equal("ui" in agents, before !== undefined); + } + }); + + it("restores a ui entry the repo already had", async () => { + const before = agents.ui; + const own = { ...ROLE_AGENTS.backend, maxIterations: 7 }; + agents.ui = own; + try { + await withUiRole(false, () => { + assert.equal("ui" in agents, false); + }); + assert.equal(agents.ui, own); + + await withUiRole(true, () => { + assert.notEqual(agents.ui, own); + }); + assert.equal(agents.ui, own); + } finally { + if (before === undefined) delete agents.ui; + else agents.ui = before; + } + }); +}); diff --git a/.sandcastle/main.mts b/.sandcastle/main.mts index 2cedf8d..796cbe0 100644 --- a/.sandcastle/main.mts +++ b/.sandcastle/main.mts @@ -69,8 +69,10 @@ // planner picked it, writes a design note the // host posts on the issue. The tester commits // failing tests, then the backend developer -// makes them pass; if either fails, the issue -// stops for the +// makes them pass, then the UI developer (when +// the planner picked it, in a repo that opted +// in) builds the Blazor components and pages; +// if any fails, the issue stops for the // round. If the branch is then ahead of main (this // run's commits or earlier ones), the host runs // scripts/gate.sh in the sandbox (checkpoint 1), @@ -80,7 +82,7 @@ // again (checkpoint 2). A red gate gets two gate-fixer // attempts per checkpoint; past that nothing is // pushed. A red checkpoint or a failed architect, -// tester or backend run is a failed build +// tester, backend or UI run is a failed build // attempt: the first gets a comment, the second // hands the issue back with sandcastle:needs-human // (lib/handback.mts). A failed scribe or reviewer diff --git a/.sandcastle/roles/backend.md b/.sandcastle/roles/backend.md index 28ccfce..96d685d 100644 --- a/.sandcastle/roles/backend.md +++ b/.sandcastle/roles/backend.md @@ -65,7 +65,7 @@ test out, or delete one. When a test is wrong rather than the code, leave it as # THE ISSUE -Your run is the last developer run: every test must pass by its end. If the task is not complete, say what was done and what remains in your last commit's body. +{{UI_DEVELOPER}} If the task is not complete, say what was done and what remains in your last commit's body. Once complete, output COMPLETE. diff --git a/.sandcastle/roles/ui.md b/.sandcastle/roles/ui.md new file mode 100644 index 0000000..769c5f8 --- /dev/null +++ b/.sandcastle/roles/ui.md @@ -0,0 +1,76 @@ +# TASK + +Make the tests pass for issue {{TASK_ID}}: {{ISSUE_TITLE}} + + + +{{ISSUE_BODY}} + + + +Comments on the issue from the repository owner: + + + +{{ISSUE_COMMENTS}} + + + +Only work on the issue specified. You can't reach GitHub from here, and don't need to: everything the issue says is above. + +Work on branch {{BRANCH}}. It may already hold earlier commits for this issue. Build on them, don't redo them. + +You are the UI developer. The tester committed failing tests for the issue's acceptance criteria, then the backend developer made every test pass except those for Blazor +components and pages. Your job is to build those components and pages, and make their tests pass. + +The architect may have written a design note for this issue at `.sandcastle/work/{{TASK_ID}}/design.md`. When it exists, read it first and follow it. Where you have to depart +from it, say what and why in your commit body, so the reviewer can check the difference is explained. + +# CONTEXT + +Here are the last 10 commits: + + + +!`git log -n 10 --format="%H%n%ad%n%B---" --date=short` + + + +# EXPLORATION + +Explore the repo and fill your context window with relevant information that will allow you to complete the task. + +Read `CLAUDE.md` for how the repo is built and laid out, `docs/adr/` (where present) for recorded decisions, and `.sandcastle/CODING_STANDARDS.md` for the code standards +(see the rules below). + +Read the tester's and the backend developer's commits on this branch first: the tests are the specification you implement, and the backend's code is what your components call. + +Use the domain terms from `CONTEXT.md` in markup, and keep pages that don't need interactivity statically rendered. + +# EXECUTION + +Work test-first (red → green → refactor), one failing test at a time: + +1. RED: pick a failing acceptance test, or write a smaller unit test of your own for a piece the acceptance tests need, and run it to watch it fail +2. GREEN: write the least implementation that passes it +3. REPEAT until every test passes +4. REFACTOR with the tests green + +Never weaken the tester's tests to make them pass: don't remove or loosen an assertion, skip or todo a test in any form (`node:test`'s `.skip`, `.todo`, +`{ skip }`, `{ todo }`, `t.skip()` or `t.todo()`; xUnit's `Skip =` or `Assert.Skip()`; +pytest's `skip`, `skipif` or `xfail` marks, or `pytest.skip()`; an early `exit 0` or `return` in a `.test.sh`), comment a +test out, or delete one. When a test is wrong rather than the code, leave it as it is and say why in your commit body. + +# RULES + +{{SHARED_RULES}} + +# THE ISSUE + +Your run is the last developer run, so the tests must pass by its end. If the task is not complete, say what was done and what remains in your last commit's body. + +Once complete, output COMPLETE. + +# FINAL RULES + +ONLY WORK ON A SINGLE TASK.