Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions .github/scripts/post_claude_review.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,16 @@

# CREDENTIAL scans from every `eyJ`, and a long run of token characters (the first segment of a JWT
# holds the letters e, y and J) is rescanned from each one: quadratic on text a diff can steer.
# So a token, with the dots joining its segments, that holds a run longer than MAX_TOKEN_RUN is
# replaced whole before CREDENTIAL looks at the text: no legitimate review text has one, and it
# leaves nothing of a long JWT (its signature included) behind. What is left has runs of at most
# MAX_TOKEN_RUN characters, which CREDENTIAL scans in time linear in the text.
# So a token, with the dots joining its segments, that holds a run longer than MAX_TOKEN_RUN and
# one of the strings every CREDENTIAL alternative starts with (CREDENTIAL_START) is replaced whole
# before CREDENTIAL looks at the text: it leaves nothing of a long JWT (its signature included)
# behind. A long token without one can't match CREDENTIAL, and legitimate review text has them (a
# generated test's file name, a long identifier, a digest), so it is left alone. What is left has
# no run holding a start longer than MAX_TOKEN_RUN characters, which CREDENTIAL scans in time
# linear in the text.
MAX_TOKEN_RUN = 128
TOKEN_CHAIN = re.compile(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*")
CREDENTIAL_START = re.compile(r"eyJ|gh[pousr]_|sk-ant-|github_pat_")


def redact_credentials(text):
Expand All @@ -96,7 +100,7 @@ def redact_credentials(text):
def long_chain(match):
nonlocal count
chain = match.group()
if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")):
if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")) and CREDENTIAL_START.search(chain):
count += 1
return REDACTED
return chain
Expand All @@ -112,9 +116,11 @@ def long_chain(match):
def redact_path(path):
"""A finding's path with credential-shaped text replaced, and how many were replaced.

Not the long-run rule: a file name is one token, and a long one (a generated test's name, a
hash-named fixture) is no credential, while a path that no longer matches the diff leaves
its finding outside it. A path past MAX_PATH is no real path and gets the full rule.
Not the long-run rule: a long name that holds one of CREDENTIAL_START's strings without being
a credential (a generated test's name with `eyJ` in it) would be replaced whole by it, and a
path that no longer matches the diff leaves its finding outside it. A long name with none of
them is left alone by redact_credentials already. A path past MAX_PATH is no real path and
gets the full rule.
"""
if len(path) > MAX_PATH:
return redact_credentials(path)
Expand Down
95 changes: 94 additions & 1 deletion .github/scripts/tests/test_post_claude_review.py
Original file line number Diff line number Diff line change
Expand Up @@ -1122,7 +1122,7 @@ def test_ordinary_text_with_short_words_is_left_alone():

def test_the_script_and_the_workflow_redact_alike():
rng = random.Random(299)
pieces = [a_jwt(), a_jwt(second=200), GLUED, "ghs_9_" + a_jwt(), "sk-ant-" + "q" * 25, "x-", "eyJ", ".", " ", "/", "a" * 130, "ok"]
pieces = [a_jwt(), a_jwt(second=200), GLUED, "ghs_9_" + a_jwt(), "sk-ant-" + "q" * 25, "x-", "eyJ", ".", " ", "/", "a" * 130, "ok", LONG_NAME]
redact_workflow = workflow_credentials()["redact_credentials"]
for _ in range(300):
text = "".join(rng.choice(pieces) for _ in range(rng.randint(1, 6)))
Expand Down Expand Up @@ -1168,6 +1168,89 @@ def test_redacting_takes_linear_time_on_long_input(text):

LONG_NAME = "tests/Get_Articles_When_" + "A" * 130 + "_Returns_401.cs"

# Long tokens that start no credential: a generated test's name, an identifier, a hex digest, a base64 blob.
LONG_PLAIN_TOKENS = {
"test name": LONG_NAME,
"identifier": "Get" + "Articles" * 30 + "ForTheCurrentUser",
"sha512 digest": "9f86d081884c7d659a2feaa0c55ad015" * 8,
"base64 blob": "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo" * 5 + "==",
"dotted name": "Company." + "Segment" * 30 + ".Service",
}


@pytest.mark.parametrize("text", LONG_PLAIN_TOKENS.values(), ids=LONG_PLAIN_TOKENS.keys())
def test_a_long_token_that_starts_no_credential_is_left_alone_with_no_warning(text):
sentence = f"The finding is in {text}, which is not a secret."
for redact_text in (pcr.redact_credentials, workflow_credentials()["redact_credentials"]):
assert redact_text(sentence) == (sentence, 0)


def test_a_long_file_name_quoted_in_a_finding_body_and_the_summary_is_left_alone():
summary, findings, count = pcr.redact(f"| {LONG_NAME} | added |", [{"path": "a.py", "line": 3, "body": f"See `{LONG_NAME}`."}])

assert LONG_NAME in summary
assert LONG_NAME in findings[0]["body"]
assert count == 0


# Each one starts a credential and holds a run over MAX_TOKEN_RUN. CREDENTIAL alone either leaves part of it behind (the ghp, ghu
# and sk-ant cases) or matches none of it (the others), so only the long-run rule replaces it whole, and each start it looks for is
# pinned.
LONG_CREDENTIAL_STARTS = {
"ghp": "ghp_" + "A" * 40 + "_" + "b" * 200,
"ghu": "ghu_" + "A" * 40 + "_" + "b" * 200,
"sk-ant": "sk-ant-" + "a" * 30 + "." + "b" * 200,
"github_pat": "github_pat_" + "a" * 300,
"eyJ in the middle": "prefix" + "q" * 140 + "eyJ" + "q" * 140,
"start in the first segment": "ghp_1." + "b" * 200,
"start in a later segment": "a" * 200 + ".eyJ" + "b" * 20,
"start before the long segment": "sk-ant-" + "a" * 5 + "." + "b" * 200,
}


@pytest.mark.parametrize("text", LONG_CREDENTIAL_STARTS.values(), ids=LONG_CREDENTIAL_STARTS.keys())
def test_a_long_token_that_starts_a_credential_is_still_replaced_whole(text):
for redact_text in (pcr.redact_credentials, workflow_credentials()["redact_credentials"]):
redacted, count = redact_text(f"see {text} end")

assert redacted == "see [redacted] end"
assert count == 1


def test_a_long_run_in_a_chain_with_no_start_is_left_alone_wherever_it_sits():
text = "see " + "x" * 200 + ".Service and Company." + "y" * 200 + " end"
for redact_text in (pcr.redact_credentials, workflow_credentials()["redact_credentials"]):
assert redact_text(text) == (text, 0)


def test_the_workflow_carries_the_scripts_long_run_rule():
workflow = workflow_credentials()

assert workflow["CREDENTIAL_START"].pattern == pcr.CREDENTIAL_START.pattern
assert workflow["MAX_TOKEN_RUN"] == pcr.MAX_TOKEN_RUN
assert workflow["TOKEN_CHAIN"].pattern == pcr.TOKEN_CHAIN.pattern
assert workflow["CREDENTIAL"].pattern == pcr.CREDENTIAL.pattern


@pytest.mark.parametrize(
"sample",
[
"sk-ant-" + "a" * 20,
*(f"gh{letter}_" + "A" * 36 for letter in "pousr"),
"ghs_123_" + a_jwt(),
"github_pat_" + "a" * 22 + "_" + "b" * 59,
a_jwt(),
],
ids=["sk-ant", "ghp", "gho", "ghu", "ghs", "ghr", "installation token", "github_pat", "jwt"],
)
def test_every_text_credential_matches_starts_with_a_credential_start(sample):
# CREDENTIAL can only match inside a token that CREDENTIAL_START finds, or the long-run rule would skip a run CREDENTIAL scans slowly.
for credential, start in ((pcr.CREDENTIAL, pcr.CREDENTIAL_START), (workflow_credentials()["CREDENTIAL"], workflow_credentials()["CREDENTIAL_START"])):
match = credential.search(sample)

assert match, sample
assert start.search(match.group())


def test_a_long_file_name_in_a_finding_path_is_left_alone():
summary, findings, count = pcr.redact("s", [{"path": LONG_NAME, "line": 3, "body": "b"}])
Expand All @@ -1176,6 +1259,16 @@ def test_a_long_file_name_in_a_finding_path_is_left_alone():
assert count == 0


def test_a_long_name_that_holds_a_credential_start_without_being_one_is_left_alone_in_a_finding_path():
path = "tests/Get_eyJ_" + "A" * 130 + "_Returns_401.cs"
_, findings, count = pcr.redact("s", [{"path": path, "line": 3, "body": "b"}])

assert findings[0]["path"] == path
assert count == 0
# The long-run rule, which a path does not get, would have replaced it whole.
assert pcr.redact_credentials(path) == ("tests/[redacted]", 1)


def test_a_credential_in_a_finding_path_is_still_redacted():
path = "src/" + a_jwt() + ".cs"
_, findings, count = pcr.redact("s", [{"path": path, "line": 3, "body": "b"}])
Expand Down
12 changes: 7 additions & 5 deletions .github/workflows/claude-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1742,7 +1742,7 @@ jobs:
# can push a change to a PR's scripts but not to this file, which a
# pull_request run takes from the PR's merge commit. It also means a PR
# cut before this check gets the check its workflow expects.
# post_claude_review.py's CREDENTIAL, MAX_TOKEN_RUN and redact_credentials must match the ones here.
# post_claude_review.py's CREDENTIAL, MAX_TOKEN_RUN, CREDENTIAL_START and redact_credentials must match the ones here.
- name: Check Claude's answer for secrets
id: answer
if: steps.secret.outputs.enabled == 'true' && steps.claude.outputs.structured_output != ''
Expand Down Expand Up @@ -1773,10 +1773,12 @@ jobs:
)
LEAK_WINDOW = 16

# Mirrors post_claude_review.py: a token holding a run longer than MAX_TOKEN_RUN is replaced
# whole before CREDENTIAL looks, so a long run of `eyJ` isn't rescanned from each one.
# Mirrors post_claude_review.py: a token holding a run longer than MAX_TOKEN_RUN and one of the
# strings CREDENTIAL starts with is replaced whole before CREDENTIAL looks, so a long run of
# `eyJ` isn't rescanned from each one. A long token without one is left alone.
MAX_TOKEN_RUN = 128
TOKEN_CHAIN = re.compile(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*")
CREDENTIAL_START = re.compile(r"eyJ|gh[pousr]_|sk-ant-|github_pat_")


def redact_credentials(text):
Expand All @@ -1785,7 +1787,7 @@ jobs:
def long_chain(match):
nonlocal count
chain = match.group()
if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")):
if any(len(run) > MAX_TOKEN_RUN for run in chain.split(".")) and CREDENTIAL_START.search(chain):
count += 1
return "[redacted]"
return chain
Expand All @@ -1796,7 +1798,7 @@ jobs:


def redact_path(path):
# Mirrors post_claude_review.py: a file name is no credential however long it is.
# Mirrors post_claude_review.py: a long name that merely holds a credential start is no credential.
if len(path) > 1024:
return redact_credentials(path)
return CREDENTIAL.subn("[redacted]", path)
Expand Down
34 changes: 26 additions & 8 deletions .sandcastle/lib/agents.mts
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,35 @@

import { readFileSync } from "node:fs";
import * as sandcastle from "@ai-hero/sandcastle";
import type { Role } from "./config.mts";
import type { AnyRole, RoleAgent } from "./config.mts";
import { ROLE_AGENTS } from "./repo-config.mts";
import { UncountedStopError } from "./errors.mts";
import { MAX_CAUSE_DEPTH, RunLimits, runLimits, usageLimitLine } from "./limits.mts";
import { usageReport } from "./report.mts";

// ROLE_AGENTS as every repo may write it: a repo-config.mts written before the
// `ui` role existed names none, and ROLE_AGENTS' own type then has no `ui` key.
const agents: Partial<Record<AnyRole, RoleAgent>> = ROLE_AGENTS;

// Whether the repo opted into the UI developer by giving ROLE_AGENTS a `ui`
// entry. A repo that hasn't never plans or runs the role.
export function uiEnabled(): boolean {
return agents.ui !== undefined;
}

// The agent and budget ROLE_AGENTS gives one role. A role the repo left out
// (only `ui` may be) is a bug in the caller, so it throws.
export function roleAgent(role: AnyRole): RoleAgent {
const agent = agents[role];
if (agent === undefined) throw new Error(`ROLE_AGENTS in lib/repo-config.mts has no ${role} entry`);
return agent;
}

// The options a role fixes; callers supply everything else.
type RoleFixed = "name" | "agent" | "maxIterations" | "signal";

export function roleOptions(role: Role, timeout: (ms: number) => AbortSignal = AbortSignal.timeout) {
const { model, effort, maxIterations, timeoutMinutes } = ROLE_AGENTS[role];
export function roleOptions(role: AnyRole, timeout: (ms: number) => AbortSignal = AbortSignal.timeout) {
const { model, effort, maxIterations, timeoutMinutes } = roleAgent(role);
return {
name: role,
agent: sandcastle.claudeCode(model, { effort }),
Expand All @@ -41,17 +59,17 @@ export function withSharedRules<T extends { prompt?: string; promptFile?: string
}

export function runRole<T>(
role: Role,
role: AnyRole,
options: Omit<sandcastle.RunOptions, RoleFixed | "output"> & { output: sandcastle.OutputObjectDefinition<T> },
limits?: RunLimits,
): Promise<sandcastle.RunResult & { output: T }>;
export function runRole(
role: Role,
role: AnyRole,
options: Omit<sandcastle.RunOptions, RoleFixed>,
limits?: RunLimits,
): Promise<sandcastle.RunResult>;
export async function runRole(
role: Role,
role: AnyRole,
options: Omit<sandcastle.RunOptions, RoleFixed>,
limits: RunLimits = runLimits,
): Promise<sandcastle.RunResult> {
Expand All @@ -67,7 +85,7 @@ export async function runRole(

export async function runRoleInSandbox(
sandbox: sandcastle.Sandbox,
role: Role,
role: AnyRole,
options: Omit<sandcastle.SandboxRunOptions, RoleFixed>,
limits: RunLimits = runLimits,
): Promise<sandcastle.SandboxRunResult> {
Expand Down Expand Up @@ -113,7 +131,7 @@ export function isRoleTimeout(error: unknown): boolean {
// records the stop, so no later role starts or publishes, and rejects with an
// UncountedStopError naming the role and the limit's line, with the original
// error as its cause. Any other error is rethrown unchanged.
export async function runWithinLimits<R>(role: Role, start: () => Promise<R>, limits: RunLimits = runLimits): Promise<R> {
export async function runWithinLimits<R>(role: AnyRole, start: () => Promise<R>, limits: RunLimits = runLimits): Promise<R> {
limits.throwIfStopped();
try {
return await start();
Expand Down
14 changes: 13 additions & 1 deletion .sandcastle/lib/agents.test.mts
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,27 @@ import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { describe, it } from "node:test";
import type { Sandbox, SandboxRunOptions } from "@ai-hero/sandcastle";
import { isRoleTimeout, roleOptions, runRoleInSandbox, runWithinLimits, withSharedRules } from "./agents.mts";
import { isRoleTimeout, roleOptions, uiEnabled, runRoleInSandbox, runWithinLimits, withSharedRules } from "./agents.mts";
import { UncountedStopError } from "./errors.mts";
import { RunLimits } from "./limits.mts";
import { ROLE_AGENTS } from "./repo-config.mts";
import { usageReport } from "./report.mts";
import { withUiRole } from "./ui-role-fixture.mts";

const sharedRules = readFileSync(new URL("../roles/shared-rules.md", import.meta.url), "utf8");

describe("roleOptions", () => {
it("builds the ui role's options from its ROLE_AGENTS entry, and refuses it when the repo has none", async () => {
await withUiRole(true, () => {
assert.equal(uiEnabled(), true);
assert.equal(roleOptions("ui").name, "ui");
});
await withUiRole(false, () => {
assert.equal(uiEnabled(), false);
assert.throws(() => roleOptions("ui"), /no ui entry/);
});
});

it("names the run after the role and applies its iteration cap", () => {
const options = roleOptions("tester");

Expand Down
23 changes: 16 additions & 7 deletions .sandcastle/lib/build.mts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { execFileSync } from "node:child_process";
import { existsSync } from "node:fs";
import { setTimeout as sleep } from "node:timers/promises";
import * as sandcastle from "@ai-hero/sandcastle";
import { isRoleTimeout, runRoleInSandbox } from "./agents.mts";
import { isRoleTimeout, runRoleInSandbox, uiEnabled } from "./agents.mts";
import { commitsAhead, fetchMain, headOfRef } from "./branches.mts";
import { changedFiles, gateFailureComment, headOf, runCheckpoint, runGate, textBlock, type Checkpoint } from "./checkpoint.mts";
import {
Expand All @@ -26,7 +26,7 @@ import { commentOnIssue, markerComments, openPullRequest, repoName, type Sandcas
import { handBackWorkflowChange, moveToLocalQueue, recordFailedAttempt, type WorkflowChange } from "./handback.mts";
import { repoGitDir, worktreeLinkProblems, worktreePathFor } from "./host-safety.mts";
import { runLimits, type RunLimits } from "./limits.mts";
import { architectPromptArgs, gateFixerPromptArgs, issuePromptArgs } from "./prompts.mts";
import { architectPromptArgs, backendPromptArgs, gateFixerPromptArgs, issuePromptArgs } from "./prompts.mts";
import type { Outcome } from "./report.mts";
import { containsSandboxSecret, containsSecret, sandboxSecrets } from "./sandbox-env.mts";
import { describeSecretMatch, firstSecretMatch, type SecretMatch } from "./scan.mts";
Expand Down Expand Up @@ -81,7 +81,7 @@ export { UncountedStopError };
// person needs to look at it regardless.
//
// `docsFailed` is true when the planner picked the scribe for this issue and
// its run failed. Unlike an architect, tester or backend failure, that
// its run failed. Unlike an architect, tester, backend or UI failure, that
// doesn't stop the pipeline (see "When a role fails" in
// Blazor-Server's docs/plans/sandcastle-workflow.md): the PR still publishes, with a note
// that the documentation step failed.
Expand Down Expand Up @@ -169,7 +169,7 @@ export async function retryOnServerError<T>(step: () => T, wait: (ms: number) =>
}
}

// The issue comment for an architect, tester or backend run that stopped
// The issue comment for an architect, tester, backend or UI run that stopped
// the build. `error` is the run's error, already made safe to post (see
// publicErrorText): it carries claude-code's stderr or last output lines, so
// it's quoted in a text block, which no mention or reference fires in, with
Expand Down Expand Up @@ -272,14 +272,14 @@ export function latestDesignNote(
}

// An issue with the optional roles the planner picked for it (see
// resolveRoles in lib/plan.mts): the architect and/or the scribe, each
// resolveRoles in lib/plan.mts): the architect, the UI developer and/or the scribe, each
// run only when listed, alongside the tester, developer and reviewer, which always run (see "Phase 6: Build" in
// Blazor-Server's docs/plans/sandcastle-workflow.md).
export type RoledIssue = SandcastleIssue & { roles: readonly OptionalRole[] };

// The roles that write the design, the tests or the code. When one fails, the
// issue stops for the round, unlike the scribe or the reviewer.
type DeveloperRole = "architect" | "tester" | "backend";
type DeveloperRole = "architect" | "tester" | "backend" | "ui";

// What buildIssue needs from outside the pipeline; tests pass stubs.
export type BuildHost = {
Expand Down Expand Up @@ -691,16 +691,24 @@ async function buildMarkedIssue(
const designNotePath = `.sandcastle/work/${issue.number}/design.md`;
const prBodyPath = `.sandcastle/work/${issue.number}/pr-body.md`;

// The UI developer runs only for an issue whose plan picked it in a repo that
// opted in. resolveRoles already keeps `ui` out of an opted-out repo's plan;
// this keeps a RoledIssue built another way from running a role the repo has
// no agent or prompt for, after a tester and backend run it can't use.
const uiRuns = issue.roles.includes("ui") && uiEnabled();

function developerPromptArgs(role: DeveloperRole): sandcastle.PromptArgs {
switch (role) {
case "architect":
return architectPromptArgs(issue, branch, host.latestDesignNote(issue.number));
case "backend":
return backendPromptArgs(issue, branch, uiRuns);
default:
return promptArgs;
}
}

// Run the architect, the tester or the developer (the backend role).
// Run the architect, the tester, the developer (the backend role) or the UI developer.
// Returns undefined when the run finished. When it threw, timed out or used
// up its iterations without signalling completion, the design, the tests or
// the code aren't done, so it returns the build's result and the issue stops
Expand Down Expand Up @@ -1098,6 +1106,7 @@ async function buildMarkedIssue(
...(issue.roles.includes("architect") ? ["architect" as const] : []),
"tester",
"backend",
...(uiRuns ? ["ui" as const] : []),
];
for (const role of developers) {
const failed = await developerFails(role);
Expand Down
Loading
Loading