diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts index 18c8611be..fc542cdf6 100644 --- a/plugins/codex-security/mcp-app/helpers-main.ts +++ b/plugins/codex-security/mcp-app/helpers-main.ts @@ -1,34 +1,48 @@ +import { existsSync, realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +export { parseCanonicalScanDraft } from "./src/artifact-scan-draft.js"; +export { resumeSelectedDeepScan } from "./src/deep-scan/finalization.js"; import { resolveSecurityMdCommand } from "./src/helpers/resolve-security-md"; import { decodePosixBytes } from "./src/helpers/posix-path"; import { windowsBinding } from "./src/native"; -let commandLine = process.argv.slice(2); -if (process.platform === "win32") { - const original = windowsBinding().windowsArguments(); - commandLine = original - .slice(original.length - commandLine.length) - .map((argument) => argument.toString("utf16le")); -} -let posixHome = process.env.HOME; -if (commandLine[0] === "--helper") { +// Importing the bundled helper from the SDK does not invoke its CLI adapter. +const entryPath = import.meta.url.startsWith("file:") ? fileURLToPath(import.meta.url) : import.meta.url; +const invokedPath = process.argv[1]; +if ( + invokedPath && existsSync(invokedPath) + && realpathSync(invokedPath) === realpathSync(entryPath) +) runHelper(); + +function runHelper(): void { + let commandLine = process.argv.slice(2); if (process.platform === "win32") { - commandLine = commandLine.slice(1); + const original = windowsBinding().windowsArguments(); + commandLine = original + .slice(original.length - commandLine.length) + .map((argument) => argument.toString("utf16le")); + } + let posixHome = process.env.HOME; + if (commandLine[0] === "--helper") { + if (process.platform === "win32") { + commandLine = commandLine.slice(1); + } else { + const [homeSet, home, ...args] = decodePosixBytes( + Buffer.from(commandLine[1] ?? "", "hex"), + ) + .split("\0") + .slice(0, -1); + posixHome = homeSet ? home : undefined; + commandLine = args; + } + } + const [command, ...args] = commandLine; + if (command === "resolve-security-md") { + process.exitCode = resolveSecurityMdCommand(args, posixHome); } else { - const [homeSet, home, ...args] = decodePosixBytes( - Buffer.from(commandLine[1] ?? "", "hex"), - ) - .split("\0") - .slice(0, -1); - posixHome = homeSet ? home : undefined; - commandLine = args; + console.error( + "Usage: launch_codex_security_mcp[.cmd] --helper resolve-security-md [options]", + ); + process.exitCode = 2; } } -const [command, ...args] = commandLine; -if (command === "resolve-security-md") { - process.exitCode = resolveSecurityMdCommand(args, posixHome); -} else { - console.error( - "Usage: launch_codex_security_mcp[.cmd] --helper resolve-security-md [options]", - ); - process.exitCode = 2; -} diff --git a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs index 94022da4f..e8e636e6b 100644 --- a/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs +++ b/plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs @@ -41,6 +41,7 @@ export async function buildMcpApp({ output }) { loader: { ".md": "text" }, logLevel: "info", logOverride: { "empty-import-meta": "silent" }, + nodePaths: [join(root, "node_modules")], outfile: bundle, platform: "node", target: "node20" @@ -108,5 +109,6 @@ const runtimeModule = new Module(loaderPath); runtimeModule.filename = loaderPath; runtimeModule.paths = Module._nodeModulePaths(dirname(loaderPath)); runtimeModule._compile(runtimeSource, loaderPath); +export default runtimeModule.exports; `; } diff --git a/plugins/codex-security/mcp-app/server.ts b/plugins/codex-security/mcp-app/server.ts index a21fbb627..257b9ba31 100644 --- a/plugins/codex-security/mcp-app/server.ts +++ b/plugins/codex-security/mcp-app/server.ts @@ -22,6 +22,7 @@ import { DeepScanStartLock, startOrJoinDeepScanCoordinator } from "./src/deep-scan/registry.js"; +import { captureDeepScanExecutionSettings, loadDeepScanExecutionSettings, restoredDeepScanWorkerSettings, type DeepScanLegacySettingsContext } from "./src/deep-scan/recovery-settings.js"; import { CodexSdkWorkerExecutor } from "./src/deep-scan/executor.js"; import { CODEX_SANDBOX_STATE_META_CAPABILITY, @@ -695,7 +696,7 @@ export function createCodexSecurityServer(): McpServer { server.registerTool("start_codex_security_deep_scan", { title: "Start or Join Codex Security Deep Scan", - description: "Run or rejoin independent Standard security scans and semantically merge their validated findings. Pass scanId and its handoffClaimToken to resume, or targetPath to start headlessly. The call blocks until the aggregate draft is ready, fails, or is canceled. On success, manifestPath identifies the canonical parent scan-manifest.json; call complete_codex_security_scan once.", + description: "Run or rejoin independent Standard security scans and semantically merge their validated findings. Pass scanId and its handoffClaimToken to resume, or targetPath to start headlessly. The call blocks until the aggregate draft is ready, fails, or is canceled. On success, manifestPath identifies the canonical parent scan-manifest.json; native scans can call complete_codex_security_scan once. SDK-managed scans leave completion to the SDK after the scan turn ends.", inputSchema: startDeepScanSchema, annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, _meta: modelActionMeta @@ -727,7 +728,13 @@ export function createCodexSecurityServer(): McpServer { return toolErrorResult(deepScanInvocationFailureMessage(error)); } const preparation = await deepScanStartLock.run(async () => { + // A joining observer does not need a usable current home. A new run, + // however, must save its original settings before creation can commit. + const executionSettings = await captureDeepScanExecutionSettings( + modelSettings, parentSandbox, process.env, { threadId, startedAt: new Date().toISOString() } + ).catch(() => null); const begun = await deepScanStore.begin({ + executionSettings, scanId, targetPath, scope: hasTarget ? scope ?? "." : undefined, @@ -743,13 +750,39 @@ export function createCodexSecurityServer(): McpServer { threadId }); } - const immediate = deepScanTerminalResult(begun.run); - if (immediate) return { begun, immediate }; + // Recipes identify SDK executions, including runs predating usageOwner. + const sdkOwned = begun.run.usageOwner?.dedicated + ?? ((await runWorkbench(["get-scan", "--scan-id", begun.run.scanId])).recipe != null); + const immediate = deepScanTerminalResult(begun.run, sdkOwned); + const completingLocally = begun.run.status === "succeeded" + && begun.run.finalizationInput + && deepScanCoordinators.get(begun.run.scanId); + const recoverableSelection = begun.run.status === "succeeded" + && begun.run.finalizationInput && !sdkOwned; + if (immediate && !completingLocally && !recoverableSelection) return { begun, immediate, sdkOwned }; const started = await startOrJoinDeepScanCoordinator({ begin: begun, registry: deepScanCoordinators, options: { store: deepScanStore, + prepareExecutor: async (run) => new CodexSdkWorkerExecutor({ + ...restoredDeepScanWorkerSettings( + await loadDeepScanExecutionSettings(run.scanDir, run, async () => { + const context = await runWorkbench(["get-scan", "--scan-id", run.scanId]); + const recipe = context.recipe as Pick | undefined; + const scan = context.scan as { executionAttribution?: { owner: DeepScanRunState["usageOwner"] } }; + return { config: recipe?.config, usageOwner: scan.executionAttribution?.owner }; + }), + parentSandbox + ), + artifactContext: { + pluginRoot: PLUGIN_ROOT, + scanRoot: run.scanDir, + repoRoot: run.targetPath, + scanId: run.scanId, + scope: run.scope + } + }), executor: new CodexSdkWorkerExecutor({ ...modelSettings, parentSandbox, @@ -765,7 +798,7 @@ export function createCodexSecurityServer(): McpServer { log: logDeepScanEvent, handoffClaimToken, threadId, - onComplete: async (draft, signal) => { + onComplete: async (draft, signal, publication) => { const context = await createScanArtifactContext( begun.run.scanId, runWorkbench, @@ -779,7 +812,25 @@ export function createCodexSecurityServer(): McpServer { await recordCodexSecurityScanDraftViaWorkbench(context, { ...draft, ...(handoffClaimToken === undefined ? {} : { handoffClaimToken }) - }, runWorkbench, signal); + }, runWorkbench, signal, publication); + }, + onFinalized: async (run, signal) => { + // The SDK stops its usage tracker and enforces the budget after this turn. + if (sdkOwned) return; + try { + await runWorkbench([ + "complete-scan", "--scan-id", run.scanId, "--thread-id", threadId, + ...optionalArg("--claim-token", handoffClaimToken), + ], undefined, false, false, signal); + } + catch (error) { + if (!signal.aborted) { + await deepScanStore.releaseCoordinator(run.scanId).catch((releaseError) => { + logDeepScanEvent({ event: "coordinator_release_failed", scanId: run.scanId, reason: boundedErrorData(releaseError).message }); + }); + } + throw new Error(deepScanInvocationFailureMessage(error), { cause: error }); + } }, onStopped: async (run) => { await runWorkbench([ @@ -791,18 +842,19 @@ export function createCodexSecurityServer(): McpServer { } } }); - return { begun, ...started }; + return { begun, sdkOwned, ...started }; }).catch((error: unknown) => ({ invocationFailure: toolErrorResult(deepScanInvocationFailureMessage(error)) })); if ("invocationFailure" in preparation) return preparation.invocationFailure; + // SDK completion belongs to the host; native selected results use the existing lease. if (preparation.immediate) return preparation.immediate; const { begun, coordinator, joined } = preparation; if (joined) { logDeepScanEvent({ event: "coordinator_joined", scanId: begun.run.scanId }); } const terminal = await coordinator.wait(abortSignalFromExtra(extra)); - const result = deepScanTerminalResult(terminal); + const result = deepScanTerminalResult(terminal, preparation.sdkOwned); if (!result) { return toolErrorResult(deepScanInvocationFailureMessage( new Error(`Deep Scan ${terminal.scanId} ended without a terminal result.`) @@ -1117,18 +1169,31 @@ export function createCodexSecurityServer(): McpServer { server.registerTool("complete_codex_security_scan", { title: "Complete Codex Security Scan", - description: "Finalization only: validate and seal already-authored scan-manifest.json, findings.json, and coverage.json, generate report.md, index findings, and mark the scan complete. For an app-backed running scan, scan-manifest.json is an unsealed draft and must omit scan.sealedAt and scan.artifacts; this tool supplies the exact workbench timestamps, seal, artifact digests, and derived finding identities. Call only after those canonical files exist; this tool does not create missing artifacts or run skipped phases. If it fails, surface the exact error and stop the current response without retrying completion or returning a final, no-findings, structured, or benchmark response.", + description: "Finalization only: validate and seal already-authored scan-manifest.json, findings.json, and coverage.json, generate report.md, index findings, and mark the scan complete. For an app-backed running scan, scan-manifest.json is an unsealed draft and must omit scan.sealedAt and scan.artifacts; this tool supplies the exact workbench timestamps, seal, artifact digests, and derived finding identities. Call only after those canonical files exist; this tool does not create missing artifacts or run skipped phases. SDK-managed Deep scans leave sealing to the SDK after the scan turn ends. If it fails, surface the exact error and stop the current response without retrying completion or returning a final, no-findings, structured, or benchmark response.", inputSchema: completeScanSchema, annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, _meta: modelActionMeta }, async ({ scanId, handoffClaimToken }) => { try { - return scanActionResult(await runWorkbench([ + const context = await runWorkbench(["get-scan", "--scan-id", scanId]); + const scan = isJsonObject(context.scan) ? context.scan : undefined; + const progress = isJsonObject(scan?.progress) ? scan.progress : undefined; + if (scan?.mode === "deep" && context.recipe != null && progress?.status === "running") { + // A tool call is still inside the owner turn; its final usage has not arrived. + return scanActionResult(redactHandoffClaimToken(context), + "The SDK completes this Deep Scan after the scan turn finishes. Continue any remaining scan work, then end the turn without calling completion again. The SDK will account for the turn, enforce its budget, and seal and publish the scan." + ); + } + const result = await runWorkbench([ "complete-scan", "--scan-id", scanId, ...optionalArg("--claim-token", handoffClaimToken) - ]), "Validated and indexed the completed Codex Security scan."); + ]); + const coverage = result.coverageSummary as JsonObject; + return scanActionResult(result, + `Validated and indexed the completed Codex Security scan. Canonical coverage: ${coverage.completeness}; ${coverage.surfaceCount} surfaces; ${coverage.deferredCount} deferred items; ${coverage.explicitExclusionCount} explicit exclusions.` + ); } catch (error) { throw new Error(completionFailureMessage(error)); } @@ -1589,13 +1654,15 @@ function boundedErrorData(error: unknown): { message: string; name: string } { }; } -function deepScanTerminalResult(run: DeepScanRunState) { +function deepScanTerminalResult(run: DeepScanRunState, sdkOwned = false) { if (run.status === "succeeded") { if (!run.manifestPath) return undefined; return { content: [{ type: "text" as const, - text: `Deep Scan discovery completed. Independent Standard scans have already performed validation and attack-path analysis and have been consolidated into the canonical scan-manifest.json, findings.json, and coverage.json under ${run.scanDir}. The returned manifestPath is the canonical scan-manifest.json, not a legacy discovery manifest. Any instructions requiring parent candidate listing, centralized validation, attack-path analysis, or another draft apply only to the old discovery-only workflow and must be skipped. The authoritative scan ID is ${run.scanId}. Immediately call complete_codex_security_scan once using that scan ID to seal and publish the scan. Return output only after completion succeeds and generated report.md exists. If completion fails, surface that exact error and return no final, no-findings, structured, or benchmark response.` + text: `Deep Scan discovery completed. Independent Standard scans have already performed validation and attack-path analysis and have been consolidated into the canonical scan-manifest.json, findings.json, and coverage.json under ${run.scanDir}. The returned manifestPath is the canonical scan-manifest.json, not a legacy discovery manifest. Any instructions requiring parent candidate listing, centralized validation, attack-path analysis, or another draft apply only to the old discovery-only workflow and must be skipped. The authoritative scan ID is ${run.scanId}. ${sdkOwned + ? "Leave the canonical artifacts unchanged and end this scan turn without calling a completion tool. The SDK will account for the completed turn, enforce its budget, and seal and publish the scan." + : "Immediately call complete_codex_security_scan once using that scan ID to seal and publish the scan. Return output only after completion succeeds and generated report.md exists. If completion fails, surface that exact error and return no final, no-findings, structured, or benchmark response."}` }], structuredContent: { manifestPath: run.manifestPath } }; @@ -1632,12 +1699,15 @@ function logDeepScanEvent(event: { async function runWorkbench( args: string[], - input?: string | Buffer + input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, ): Promise { let pythonCommand: string | undefined; try { pythonCommand = await resolvePythonCommand(); - return await executeWorkbenchWithStateSelection(pythonCommand, args, input); + return await executeWorkbenchWithStateSelection(pythonCommand, args, input, selectFinalization, withExecutionSettings, signal); } catch (error) { const launchError = pythonCommand ? missingPythonHelperMessage(error, pythonCommand) @@ -1655,36 +1725,39 @@ async function runWorkbench( async function executeWorkbenchWithStateSelection( pythonCommand: string, args: string[], - input?: string | Buffer + input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, ): Promise { if (WORKBENCH_COMMANDS_WITHOUT_DATABASE.has(args[0] ?? "")) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings, signal); } if (CONFIGURED_WORKBENCH_STATE_DIR) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings, signal); } if (fallbackWorkbenchStateDir) { - return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input); + return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input, selectFinalization, withExecutionSettings, signal); } if (persistentWorkbenchStateSucceeded) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings, signal); } return await withWorkbenchStateSelectionLock(async () => { if (fallbackWorkbenchStateDir) { - return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input); + return await executeWorkbench(pythonCommand, args, await fallbackWorkbenchStateDir, input, selectFinalization, withExecutionSettings, signal); } if (persistentWorkbenchStateSucceeded) { - return await executeWorkbench(pythonCommand, args, undefined, input); + return await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings, signal); } try { - const result = await executeWorkbench(pythonCommand, args, undefined, input); + const result = await executeWorkbench(pythonCommand, args, undefined, input, selectFinalization, withExecutionSettings, signal); persistentWorkbenchStateSucceeded = true; return result; } catch (error) { if (!isUnwritableSqliteOpenError(error)) throw error; const fallbackStateDir = await pinFallbackWorkbenchStateDir(); logWorkbenchStateFallback(); - return await executeWorkbench(pythonCommand, args, fallbackStateDir, input); + return await executeWorkbench(pythonCommand, args, fallbackStateDir, input, selectFinalization, withExecutionSettings, signal); } }); } @@ -1707,7 +1780,10 @@ async function executeWorkbench( pythonCommand: string, args: string[], stateDir?: string, - input?: string | Buffer + input?: string | Buffer, + selectFinalization = false, + withExecutionSettings = false, + signal?: AbortSignal, ): Promise { const userContextIndex = args.indexOf("--user-context"); const userContext = userContextIndex === -1 ? undefined : args[userContextIndex + 1]; @@ -1716,7 +1792,13 @@ async function executeWorkbench( workbenchArgs.splice(userContextIndex, 2, "--user-context-stdin"); } const workbenchInput = input ?? userContext; - const execution = execFileAsync(pythonCommand, [workbenchScriptPath(), ...workbenchArgs], { + const internalInvocation = selectFinalization ? "select_finalization=True" + : withExecutionSettings ? "with_execution_settings=True" : undefined; + const pythonArgs = internalInvocation + ? ["-c", `import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](${internalInvocation})`, workbenchScriptPath(), ...workbenchArgs] + : [workbenchScriptPath(), ...workbenchArgs]; + const execution = execFileAsync(pythonCommand, pythonArgs, { + signal, cwd: PLUGIN_ROOT, env: stateDir ? { ...process.env, CODEX_SECURITY_STATE_DIR: stateDir } diff --git a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts index b8e147ed9..7749d0ceb 100644 --- a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts +++ b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts @@ -1,4 +1,4 @@ -import { join } from "node:path"; +import { dirname, join, relative, sep } from "node:path"; import type { ZodType } from "zod/v4"; import commonSchema from "../../schemas/definitions/artifact-common.schema.json"; import reducerSchema from "../../schemas/tools/deep-reducer.schema.json"; @@ -20,7 +20,9 @@ import { type DeepScanArtifacts } from "./deep-scan/artifacts.js"; import { + deepReductionForPersistence, parseDeepReduction, + projectDiscoveryCoverage, reconcileDeepReduction, type DeepReductionInput, type DeepReductionSources, @@ -54,6 +56,18 @@ interface BoundReducer { /** Read the findings and scan context assigned to this reducer. */ export async function getCodexSecurityDeepReducerInputs( context: ArtifactContext +): Promise { + const inputs = await readDeepReductionSources(context); + const { sourceCoverage: _coverage, ...previous } = inputs.previous ?? {}; + return { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ workerId, result })), + previous: inputs.previous === null ? null : previous as DeepReductionInput, + }; +} + +/** Capture host coverage alongside the reducer's immutable finding inputs. */ +export async function readDeepReductionSources( + context: ArtifactContext ): Promise { return withLogicalReducerErrors(context, async () => { const bound = bindDeepReducer(context); @@ -73,8 +87,13 @@ export async function getCodexSecurityDeepReducerInputs( sourceFindingIds: [`${worker.id}:${index}`], }, })); - const { coverage: _coverage, ...reduction } = result; - return { workerId: worker.id, result: reduction }; + const { coverage, ...reduction } = result; + return { + workerId: worker.id, + ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }), + coverage: projectDiscoveryCoverage(coverage, worker, relative(bound.artifacts.scanDir, worker.artifactDir ?? dirname(worker.resultPath)).split(sep).join("/")), + result: reduction, + }; })); const previous = await readPreviousReduction(bound); const scanId = bound.scanId ?? previous?.scanId ?? discoveries[0]?.result.scanId; @@ -107,7 +126,7 @@ export async function recordCodexSecurityDeepReduction( const submitted = deepReductionInputSchema.parse(input); let reduction = parseDeepReduction(submitted); if (reduction.complete === false) throw new Error("Deep reduction is only a checkpoint, not a complete result."); - const inputs = await getCodexSecurityDeepReducerInputs(context); + const inputs = await readDeepReductionSources(context); const expectedScanId = bound.scanId ?? inputs.previous?.scanId ?? inputs.discoveries[0]?.result.scanId; @@ -116,8 +135,9 @@ export async function recordCodexSecurityDeepReduction( } reduction = reconcileDeepReduction(reduction, inputs.discoveries, inputs.previous); - await saveScanDraftCheckpoint(context, reduction); - await writeJsonAtomic(bound.resultPath, reduction); + const persisted = deepReductionForPersistence(reduction, bound.state.persistSourceCoverage); + await saveScanDraftCheckpoint(context, persisted); + await writeJsonAtomic(bound.resultPath, persisted); return { findingCount: reduction.findings.length, consumedWorkerIds: bound.state.claimedWorkers.map((worker) => worker.id) diff --git a/plugins/codex-security/mcp-app/src/artifact-io.ts b/plugins/codex-security/mcp-app/src/artifact-io.ts index 3208f6d8a..66a7e8f38 100644 --- a/plugins/codex-security/mcp-app/src/artifact-io.ts +++ b/plugins/codex-security/mcp-app/src/artifact-io.ts @@ -5,12 +5,16 @@ import { dirname, isAbsolute, join, resolve, sep } from "node:path"; export interface DeepReducerWorkerContext { id: string; resultPath: string; + /** Original output owner for relative evidence, including accepted checkpoints. */ + artifactDir?: string; + attempt?: number; } export interface DeepReducerContext { scanRoot: string; claimedWorkers: DeepReducerWorkerContext[]; previousReducerResultPath?: string; + persistSourceCoverage?: boolean; } /** diff --git a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts index 4c560822c..651ff1f10 100644 --- a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts +++ b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts @@ -4,6 +4,7 @@ import { dirname, join, sep } from "node:path"; import type * as z from "zod/v4"; import commonSchema from "../../schemas/definitions/artifact-common.schema.json"; import scanDraftDocument from "../../schemas/tools/scan-draft.schema.json"; +import scanManifestDocument from "../../schemas/scan-manifest.schema.json"; import type { ArtifactContext } from "./artifact-context.js"; import type { RunArtifactWorkbench } from "./artifact-context.js"; import { @@ -17,17 +18,10 @@ import { type SchemaDocument, } from "./artifact-schema-loader.js"; -type JsonObject = Record; +import type { ScanDraftInput } from "../../../../sdk/typescript/src/accepted-audit.js"; +export type { ScanDraftInput } from "../../../../sdk/typescript/src/accepted-audit.js"; -export interface ScanDraftInput { - scanId: string; - complete?: boolean; - handoffClaimToken?: string; - scope?: JsonObject; - threatModel?: JsonObject; - findings: JsonObject[]; - coverage: JsonObject; -} +type JsonObject = Record; export interface CompletedScanInput { scanId: string; @@ -55,6 +49,12 @@ interface PreparedScanDraft { coverage: JsonObject; } +/** Host-selected Deep aggregate, separate from model-authored draft fields. */ +export interface DeepScanPublication { + coordinatorGeneration?: number; + resultPath: string | null; +} + type PublishScanDraft = ( draft: PreparedScanDraft, expectedDigest: string | undefined, @@ -69,6 +69,19 @@ export const scanDraftInputSchema = loadArtifactZodSchema( "scanDraftInput", ) as z.ZodType; +// Sealed documents retain the existing public ID contract; live drafts require UUIDs. +const canonicalScanDraftInputSchema = loadArtifactZodSchema( + [commonSchema, { + ...scanDraftDocument, + $defs: { + ...scanDraftDocument.$defs, + scanId: scanManifestDocument.properties.scan.properties.id, + }, + }] as SchemaDocument[], + scanDraftDocument.$id, + "scanDraftInput", +) as z.ZodType; + export const completedScanInputSchema = loadArtifactZodSchema( schemaDocuments, scanDraftDocument.$id, @@ -165,6 +178,7 @@ export async function recordCodexSecurityScanDraftViaWorkbench( input: ScanDraftInput, runWorkbench: RunArtifactWorkbench, signal?: AbortSignal, + publication?: DeepScanPublication, ): Promise { return recordCodexSecurityScanDraft( context, @@ -184,7 +198,10 @@ export async function recordCodexSecurityScanDraftViaWorkbench( const { handoffClaimToken: _claim, ...snapshot } = checkpoint; await Promise.all([ replaceArtifactJson(checkpointPath, snapshot), - replaceArtifactJson(draftPath, draft), + replaceArtifactJson(draftPath, { + ...draft, + ...(publication === undefined ? {} : { deepScanPublication: publication }), + }), ]); const arguments_ = [ "write-scan-draft", @@ -548,32 +565,9 @@ async function readPreviousScanDraft( const manifest = parseJsonObject(contents[0]!, "previous scan draft manifest"); const findings = parseJsonObject(contents[1]!, "previous scan draft findings"); const coverage = parseJsonObject(contents[2]!, "previous scan draft coverage"); - const scan = requireObject(manifest.scan, "previous scan draft.scan"); - const semanticScope = isObject(scan.scope) ? { ...scan.scope } : undefined; - if (semanticScope) { - delete semanticScope.includePaths; - delete semanticScope.excludePaths; - } - const semanticCoverage = { ...coverage }; - for (const field of ["documentType", "schemaVersion", "scanId", "mode", "includePaths", "excludePaths", "receiptRefs", "inventoryStrategy"]) delete semanticCoverage[field]; return { digest, - input: parsePersistedScanDraft({ - scanId: context.scanId, - ...(scan.complete === false ? { complete: false } : {}), - ...(semanticScope && Object.keys(semanticScope).length > 0 - ? { scope: semanticScope } - : {}), - ...(isObject(scan.threatModel) - ? { threatModel: structuredClone(scan.threatModel) } - : {}), - findings: (findings.findings as JsonObject[]).map((finding) => { - const semantic = { ...finding }; - for (const field of ["findingId", "occurrenceId", "fingerprints"]) delete semantic[field]; - return semantic; - }), - coverage: semanticCoverage, - }), + input: parseCanonicalScanDraft({ scanId: context.scanId, manifest, findings, coverage }), }; } @@ -973,8 +967,35 @@ export async function getCodexSecurityCompletedScan( return { scanId: parsed.scanId, manifest, findings, coverage }; } +/** Project canonical documents through the same semantic parser as worker drafts. */ +export function parseCanonicalScanDraft(input: { + scanId?: string; + manifest: JsonObject; + findings: JsonObject; + coverage: JsonObject; +}): ScanDraftInput { + const scan = requireObject(input.manifest.scan, "scan draft manifest.scan"); + for (const scanId of [scan.id, input.findings.scanId, input.coverage.scanId]) { + if (scanId !== undefined && scanId !== input.scanId) { + throw new Error("scan draft: canonical documents belong to a different scan."); + } + } + return parsePersistedCheckpoint({ + scanId: input.scanId, + ...(scan.complete === undefined ? {} : { complete: scan.complete }), + ...(scan.scope === undefined ? {} : { scope: scan.scope }), + ...(scan.threatModel === undefined ? {} : { threatModel: scan.threatModel }), + findings: input.findings.findings, + coverage: input.coverage, + }, canonicalScanDraftInputSchema); +} + export function parseScanDraft(input: ScanDraftInput): ScanDraftInput { - const parsed = scanDraftInputSchema.parse(input); + return parseSemanticScanDraft(input, scanDraftInputSchema); +} + +function parseSemanticScanDraft(input: unknown, schema: z.ZodType): ScanDraftInput { + const parsed = schema.parse(input); validateFindingSemantics(parsed.findings); validateCoverageSemantics(parsed.coverage); return parsed; @@ -984,18 +1005,23 @@ export function parseScanDraft(input: ScanDraftInput): ScanDraftInput { export function parsePersistedScanDraft( input: Record ): ScanDraftInput { + return parsePersistedDraft(input, scanDraftInputSchema); +} + +function parsePersistedDraft(input: Record, schema: z.ZodType): ScanDraftInput { const compatible = structuredClone(input); - if (!Array.isArray(compatible.findings)) { - return parseScanDraft(compatible as unknown as ScanDraftInput); - } - for (const finding of compatible.findings) { - if (!isObject(finding)) continue; - normalizePersistedFindingDetails(finding); + if (Array.isArray(compatible.findings)) { + for (const finding of compatible.findings) { + if (isObject(finding)) normalizePersistedFindingDetails(finding); + } } - return parseScanDraft(compatible as unknown as ScanDraftInput); + return parseSemanticScanDraft(compatible, schema); } -function parsePersistedCheckpoint(input: Record): ScanDraftInput { +function parsePersistedCheckpoint( + input: Record, + schema = scanDraftInputSchema, +): ScanDraftInput { const compatible = structuredClone(input); if (isObject(compatible.scope)) { delete compatible.scope.includePaths; @@ -1022,7 +1048,7 @@ function parsePersistedCheckpoint(input: Record): ScanDraftInpu delete finding.fingerprints; } } - return parsePersistedScanDraft(compatible); + return parsePersistedDraft(compatible, schema); } function normalizePersistedFindingDetails(finding: JsonObject): void { diff --git a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts index 291557a9e..e6143a84a 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts @@ -1,3 +1,4 @@ +import { join } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { parsePersistedScanDraft, @@ -10,10 +11,13 @@ import { import { readJsonObject, requireRegularFile, writeJsonAtomic } from "./artifacts.js"; import type { DeepScanArtifacts } from "./artifacts.js"; -export type DeepReductionInput = Omit; +export type DeepReductionInput = Omit & { + /** Host projection of accepted source coverage; never supplied by the reducer. */ + sourceCoverage?: ScanDraftInput["coverage"]; +}; export interface DeepReductionSources { - discoveries: { workerId: string; result: DeepReductionInput }[]; + discoveries: { workerId: string; attempt?: number; coverage?: ScanDraftInput["coverage"]; result: DeepReductionInput }[]; previous: DeepReductionInput | null; } @@ -22,6 +26,21 @@ export interface ReducerArtifactValidation { result: DeepReductionInput; } +export function deepReductionToScanDraft(result: DeepReductionInput): ScanDraftInput { + const { sourceCoverage, ...draft } = structuredClone(result); + return { ...draft, coverage: sourceCoverage ?? unknownSourceCoverage() }; +} + +/** Older workflow readers reject the host field; retain their persisted shape. */ +export function deepReductionForPersistence( + result: DeepReductionInput, + persistSourceCoverage = false, +): DeepReductionInput { + if (persistSourceCoverage) return result; + const { sourceCoverage: _coverage, ...legacy } = result; + return legacy; +} + /** * Check reducer findings with the Standard scan validator. * It requires coverage, so add an empty value and remove it after validation. @@ -30,8 +49,9 @@ export function parseDeepReduction( input: Record, persisted = false, ): DeepReductionInput { + const { sourceCoverage, ...submitted } = input; const standard = { - ...input, + ...submitted, coverage: { completeness: "complete", surfaces: [], @@ -42,6 +62,9 @@ export function parseDeepReduction( const { coverage: _coverage, ...parsed } = persisted ? parsePersistedScanDraft(standard) : parseScanDraft(standard as unknown as ScanDraftInput); + if (persisted && sourceCoverage !== undefined) { + return { ...parsed, sourceCoverage: parsePersistedScanDraft({ ...standard, coverage: sourceCoverage }).coverage }; + } return parsed; } @@ -50,6 +73,16 @@ export async function validateDiscoveryArtifacts( artifacts: DeepScanArtifacts, resultPath: string, expectedScanId: string +): Promise { + const result = await readDiscoveryAuditDraft(artifacts, resultPath, expectedScanId); + if (result.complete === false) throw new Error("Standard scan worker wrote only a checkpoint; its audit is not complete."); + return result; +} + +export async function readDiscoveryAuditDraft( + artifacts: DeepScanArtifacts, + resultPath: string, + expectedScanId: string, ): Promise { await requireRegularFile(resultPath, artifacts.workersRoot); const result = parseStoredScanDraft( @@ -58,10 +91,28 @@ export async function validateDiscoveryArtifacts( expectedScanId, parsePersistedScanDraft ); - if (result.complete === false) throw new Error("Standard scan worker wrote only a checkpoint; its audit is not complete."); + if (result.complete !== false) await validateWriteupFiles(artifacts, result); return result; } +async function validateWriteupFiles( + artifacts: DeepScanArtifacts, + result: Pick, +): Promise { + for (const [index, finding] of result.findings.entries()) { + const reportPath = (finding.writeup as { reportPath: string } | undefined)?.reportPath; + if (!reportPath) continue; + try { + await requireRegularFile(join(artifacts.scanDir, reportPath), artifacts.scanDir, false); + } catch (cause) { + throw new Error( + `findings[${index}].writeup.reportPath: expected a file inside the scan directory: ${reportPath}`, + { cause }, + ); + } + } +} + /** Validate the complete aggregate and derive convergence from stable finding identities. */ export async function validateReducerArtifacts(input: { artifacts: DeepScanArtifacts; @@ -70,6 +121,7 @@ export async function validateReducerArtifacts(input: { reducerId: string; previousReducerResultPath?: string; sources?: DeepReductionSources; + persistSourceCoverage?: boolean; }, expectedScanId?: string): Promise { const { artifacts, @@ -101,11 +153,15 @@ export async function validateReducerArtifacts(input: { if (input.sources) { result = reconcileDeepReduction(result, input.sources.discoveries, input.sources.previous); - await saveScanDraftCheckpoint({ root: artifactDir, repoRoot: artifacts.scanDir, layout: "reducer" }, result); - await writeJsonAtomic(resultPath, result); } else { validateRetainedFindings(result, [], previous); } + await validateWriteupFiles(artifacts, result); + if (input.sources) { + const persisted = deepReductionForPersistence(result, input.persistSourceCoverage); + await saveScanDraftCheckpoint({ root: artifactDir, repoRoot: artifacts.scanDir, layout: "reducer" }, persisted); + await writeJsonAtomic(resultPath, persisted); + } const previousFindingIds = new Set((previous?.findings ?? []).map(scanFindingIdentity)); return { result, @@ -122,6 +178,7 @@ export function reconcileDeepReduction( previous: DeepReductionInput | null, ): DeepReductionInput { const result = structuredClone(input); + result.sourceCoverage = aggregateSourceCoverage(discoveries, previous); if (result.complete === false) throw new Error("Deep reduction is only a checkpoint, not a complete result."); for (const source of [...discoveries.map((discovery) => discovery.result), ...(previous ? [previous] : [])]) { if (source.scanId !== result.scanId) throw new Error("Deep reduction source belongs to a different scan."); @@ -180,6 +237,85 @@ export function reconcileDeepReduction( return result; } +/** Keep independent reviews separate: matching labels do not resolve another pass's proof gap. */ +export function aggregateSourceCoverage( + discoveries: DeepReductionSources["discoveries"], + previous: DeepReductionInput | null, +): ScanDraftInput["coverage"] { + const sources = [ + ...(previous ? [previous.sourceCoverage ?? unknownSourceCoverage()] : []), + ...discoveries.map((source) => source.coverage ?? unknownSourceCoverage()), + ]; + const result: ScanDraftInput["coverage"] = { + completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], reviews: [], + }; + for (const field of ["surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"]) { + const entries = sources.flatMap((source) => (source[field] as unknown[] | undefined) ?? []); + if (entries.length || field !== "openQuestions") result[field] = structuredClone(entries); + } + if (sources.some((source) => source.completeness === "partial") + || (result.deferred as unknown[]).length > 0 + || (result.surfaces as Record[]).some((surface) => surface.disposition === "needs_follow_up")) { + result.completeness = "partial"; + } else if (sources.some((source) => source.completeness === "unknown")) { + result.completeness = "unknown"; + } + return result; +} + +function unknownSourceCoverage(): ScanDraftInput["coverage"] { + return { completeness: "unknown", surfaces: [], explicitExclusions: [], deferred: [] }; +} + +/** Qualify worker-local IDs and receipt paths before combining accepted coverage. */ +export function projectDiscoveryCoverage( + coverage: ScanDraftInput["coverage"], + worker: { id: string; attempt?: number }, + artifactPrefix: string, +): ScanDraftInput["coverage"] { + const provenance = { workerId: worker.id, ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }) }; + const prefix = `${worker.id}-attempt-${worker.attempt ?? "unknown"}`; + const surfaces = coverage.surfaces as Record[]; + const surfaceIds = new Map(surfaces.map((surface, index) => [surface.id, `${prefix}-surface-${index + 1}`])); + const project = (item: Record) => { + const projected = structuredClone(item); + const sourceProvenance = typeof projected.provenance === "object" && projected.provenance !== null + && !Array.isArray(projected.provenance) ? projected.provenance as Record : {}; + // Preserve source descriptions while deriving identity only from the accepted owner. + for (const key of ["workerId", "attempt", "sourceId", "candidateId"]) delete sourceProvenance[key]; + projected.provenance = { + ...sourceProvenance, + ...provenance, + ...(item.id === undefined ? {} : { sourceId: item.id }), + ...(item.candidateId === undefined ? {} : { candidateId: item.candidateId }), + }; + return projected; + }; + return { + completeness: coverage.completeness, + reviews: [{ ...provenance, completeness: coverage.completeness }], + surfaces: surfaces.map((surface, index) => ({ + ...project(surface), + id: `${prefix}-surface-${index + 1}`, + receiptRefs: ((surface.receiptRefs as string[] | undefined) ?? []).map((ref) => `${artifactPrefix}/${ref}`), + })), + explicitExclusions: (coverage.explicitExclusions as Record[]).map(project), + deferred: (coverage.deferred as Record[]).map((item, index) => ({ + ...project(item), + id: `${prefix}-deferred-${index + 1}`, + ...(item.candidateId === undefined ? {} : { candidateId: `${prefix}-candidate-${index + 1}` }), + ...(item.surfaceIds === undefined ? {} : { + surfaceIds: (item.surfaceIds as string[]).map((id) => surfaceIds.get(id) ?? id), + }), + })), + ...(coverage.openQuestions === undefined ? {} : { + openQuestions: (coverage.openQuestions as (string | Record)[]).map((question) => ( + project(typeof question === "string" ? { question } : question) + )), + }), + }; +} + function findingSourceIds(finding: Record): string[] { const provenance = finding.provenance as Record; const ids = provenance.sourceFindingIds; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts index 905c0c9a8..d1b6101c0 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts @@ -1,26 +1,23 @@ +import { publishSelectedDeepScan } from "./finalization.js"; import { randomUUID } from "node:crypto"; -import { promises as fs } from "node:fs"; import { basename, dirname, join } from "node:path"; import { createDeepScanArtifacts, ensureDeepScanDirectories } from "./artifacts.js"; -import { validateDiscoveryArtifacts, validateReducerArtifacts, type DeepReductionInput } from "./artifact-validation.js"; +import { aggregateSourceCoverage, deepReductionToScanDraft, validateDiscoveryArtifacts, validateReducerArtifacts, type DeepReductionInput } from "./artifact-validation.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { scanDraftInputSchema, + type DeepScanPublication, type ScanDraftInput } from "../artifact-scan-draft.js"; import type { DeepScanArtifacts } from "./artifacts.js"; -import { - DeepScanWorkerRunner, - sha256 -} from "./worker-runner.js"; +import { DeepScanWorkerRunner } from "./worker-runner.js"; import type { AcceptedDiscovery, DedupOutcome, - DiscoveryOutcome, - SuccessfulDedupOutcome, - WorkerExecutionAudit + DiscoveryOutcome } from "./worker-runner.js"; import { boundedDeepScanErrorPair, @@ -48,30 +45,15 @@ type SchedulerSettlement = | { status: "fulfilled"; outcome: SchedulerOutcome } | { status: "rejected"; error: unknown }; -type AcceptedReducer = Omit; - interface SchedulerResult { reason: DeepScanTerminalReason; omittedWorkerIds: string[]; - canceledWorkerIds: string[]; - accepted: AcceptedDiscovery[]; - mergedWorkerIds: string[]; - reducers: AcceptedReducer[]; result?: DeepReductionInput; + resultPath?: string; } type CoordinatorPhase = "setup" | "discovery" | "terminal"; -interface SchedulerAudit { - accepted: AcceptedDiscovery[]; - mergedWorkerIds: string[]; - omittedWorkerIds: string[]; - canceledWorkerIds: string[]; - bufferedWorkerIds: string[]; - reducers: AcceptedReducer[]; - executions: WorkerExecutionAudit[]; -} - export interface CoordinatorOptions { run: DeepScanRunState; store: DeepScanStore; @@ -86,7 +68,9 @@ export interface CoordinatorOptions { threadId?: string; heartbeatIntervalMs?: number; observeReplacement?: (run: DeepScanRunState) => Promise; - onComplete?: (draft: ScanDraftInput, signal: AbortSignal) => Promise; + onComplete?: (draft: ScanDraftInput, signal: AbortSignal, publication: DeepScanPublication) => Promise; + /** Complete the enclosing scan after selected publication, before local waiters settle. */ + onFinalized?: (run: DeepScanRunState, signal: AbortSignal) => Promise; onStopped?: (run: DeepScanRunState) => Promise; } @@ -124,15 +108,6 @@ export class DeepScanCoordinator { private externallyFailed = false; private phase: CoordinatorPhase = "setup"; private discoveryDeadlineReached = false; - private readonly audit: SchedulerAudit = { - accepted: [], - mergedWorkerIds: [], - omittedWorkerIds: [], - canceledWorkerIds: [], - bufferedWorkerIds: [], - reducers: [], - executions: [] - }; private state: DeepScanRunState; constructor(private readonly options: CoordinatorOptions) { @@ -149,10 +124,7 @@ export class DeepScanCoordinator { clock: this.clock, random: options.random ?? Math.random, log: this.log, - retryDelaysMs: options.retryDelaysMs ?? RETRY_DELAYS_MS, - recordExecution: (execution) => { - this.audit.executions.push(execution); - } + retryDelaysMs: options.retryDelaysMs ?? RETRY_DELAYS_MS } satisfies Omit[0], "signal">; this.workers = new DeepScanWorkerRunner({ ...workerOptions, @@ -175,7 +147,7 @@ export class DeepScanCoordinator { if (this.started) return; this.started = true; this.log({ event: "coordinator_started", scanId: this.state.scanId }); - this.scheduleDiscoveryDeadline(); + if (!this.state.finalizationInput) this.scheduleDiscoveryDeadline(); this.scheduleHeartbeat(); void this.run().catch((error: unknown) => { this.log({ @@ -288,22 +260,29 @@ export class DeepScanCoordinator { await ensureDeepScanDirectories(this.artifacts); if (this.canceled || this.externallyFailed) return; + if (this.state.finalizationInput) { + this.phase = "terminal"; + await this.completeSelectedFinalization(); + return; + } this.phase = "discovery"; const schedulerResult = await this.runScheduler(); if (this.canceled || this.externallyFailed) return; this.phase = "terminal"; + if (this.state.workflowVersion === "deep-security-scan/v2") { + if (!this.options.store.selectFinalization) throw new Error("The Deep Scan store cannot select finalization input."); + this.state = await this.options.store.selectFinalization({ + scanId: this.state.scanId, + reason: schedulerResult.reason, + manifestPath: join(this.state.scanDir, "scan-manifest.json"), + resultPath: schedulerResult.resultPath, + omittedWorkerIds: schedulerResult.omittedWorkerIds, + }); + await this.completeSelectedFinalization(); + return; + } const draft = schedulerResult.result - ? { - ...structuredClone(schedulerResult.result), - // Readers require coverage.json. The coordinator has accepted this - // result, so mark it complete and leave review notes empty. - coverage: { - completeness: "complete", - surfaces: [], - explicitExclusions: [], - deferred: [] - } - } + ? deepReductionToScanDraft(schedulerResult.result) : scanDraftInputSchema.parse({ scanId: this.state.scanId, findings: [], @@ -319,9 +298,17 @@ export class DeepScanCoordinator { if (draft.scanId !== this.state.scanId) { throw new Error("Deep Scan aggregate does not match its authoritative scan identity."); } - await this.options.onComplete?.(draft, this.publicationAbortController.signal); + await this.options.onComplete?.(draft, this.publicationAbortController.signal, { + coordinatorGeneration: this.state.coordinatorGeneration, + resultPath: schedulerResult.resultPath ?? null, + }); if (this.canceled || this.externallyFailed) return; - this.state = await this.finishWithReplay(schedulerResult); + this.state = await this.options.store.finish({ + scanId: this.state.scanId, + reason: schedulerResult.reason, + manifestPath: join(this.state.scanDir, "scan-manifest.json"), + omittedWorkerIds: schedulerResult.omittedWorkerIds, + }); if (this.canceled || this.externallyFailed) return; this.log({ event: "coordinator_terminal", @@ -341,6 +328,12 @@ export class DeepScanCoordinator { await this.settleSchedulerWork(); return; } + if (this.state.finalizationInput) { + // Publication can be retried from the committed input without model work. + this.log({ event: "coordinator_publication_pending", scanId: this.state.scanId, reason: errorKind(error) }); + this.failLocally(error); + return; + } const message = errorMessage(error); const persistedMessage = boundedDeepScanErrorMessage(error); if (this.phase === "setup") { @@ -437,6 +430,20 @@ export class DeepScanCoordinator { } } + private async completeSelectedFinalization(): Promise { + this.state = await publishSelectedDeepScan({ + run: this.state, + artifacts: this.artifacts, + signal: this.publicationAbortController.signal, + publish: async (...args) => { await this.options.onComplete?.(...args); }, + finish: (input) => this.options.store.finish(input), + }); + if (this.canceled || this.externallyFailed) return; + await this.options.onFinalized?.(cloneState(this.state), this.publicationAbortController.signal); + if (this.canceled || this.externallyFailed) return; + this.finishLocally(this.state); + } + private finishLocally(state: DeepScanRunState): void { if (this.terminal) return; this.terminal = true; @@ -560,7 +567,17 @@ export class DeepScanCoordinator { && this.state.coordinatorGeneration !== undefined && current.coordinatorGeneration > this.state.coordinatorGeneration ); - if (current.status === "running" && !replacementConfirmed) return false; + // finish-deep-scan commits before the enclosing scan's finalizer. Its own + // succeeded generation is still ours until that finalizer settles. + const completingSelectedParent = current.status === "succeeded" + && this.state.finalizationInput !== undefined + && current.coordinatorGeneration === this.state.coordinatorGeneration; + if (!replacementConfirmed && (current.status === "running" || completingSelectedParent)) { + // A selection response can be lost after its transaction commits. + if (current.finalizationInput) this.state = { ...this.state, + finalizationInput: current.finalizationInput, terminalReason: current.terminalReason }; + return false; + } this.externallyFailed = true; this.abortController.abort("deep_scan_coordinator_lease_lost"); @@ -603,23 +620,12 @@ export class DeepScanCoordinator { .filter((worker) => worker.kind === "discovery" && worker.mergeState === "merged") .map((worker) => worker.id) ); - const mergedDiscoveries: AcceptedDiscovery[] = recovered.filter((worker) => ( - mergedIds.has(worker.id) - )); - const canceledWorkerIds = (this.state.persistedWorkers ?? []) - .filter((worker) => worker.kind === "discovery" && worker.status === "canceled") - .map((worker) => worker.id); const omittedWorkerIds: string[] = []; - this.audit.accepted = [...accepted]; - this.audit.mergedWorkerIds = mergedDiscoveries.map((worker) => worker.id); - this.audit.canceledWorkerIds = [...canceledWorkerIds]; - this.audit.executions = await this.recoverPersistedExecutions(); const recoveredReducers = await this.recoverCompletedReducers(recovered); - const reducerOutcomes = recoveredReducers.reducers; let latestResult = recoveredReducers.result; let buffer: AcceptedDiscovery[] = recovered.filter((worker) => !mergedIds.has(worker.id)); let reducer: Promise | undefined; - let previousReducerResultPath = reducerOutcomes.at(-1)?.resultPath; + let previousReducerResultPath = recoveredReducers.resultPath; let dispatched = this.state.dispatchedCount; let workerSequence = Math.max( dispatched, @@ -636,8 +642,6 @@ export class DeepScanCoordinator { let stopReason: DeepScanTerminalReason | undefined; let lastReplaceableFailure: Extract | undefined; - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); - this.audit.reducers = [...reducerOutcomes]; const errorLimit = config.stopAfterConsecutiveErrors ?? config.stopAfterNoNew; let reducerFailures = persistedReducerFailureStreak(this.state.persistedWorkers ?? []); if (this.state.consecutiveErrors >= errorLimit) { @@ -707,16 +711,12 @@ export class DeepScanCoordinator { const workerId = entries[index]?.[0]; if (workerId) active.delete(workerId); if (result.status === "rejected") { - if (workerId) removeValue(canceledWorkerIds, workerId); firstFailure ??= result.reason; continue; } const outcome = result.value; if (outcome.status === "failed") { - if (outcome.replaceableFailureKind) { - canceledWorkerIds.push(outcome.workerId); - } else { - removeValue(canceledWorkerIds, outcome.workerId); + if (!outcome.replaceableFailureKind) { firstFailure ??= outcome.error; } } else if (outcome.status === "succeeded") { @@ -728,15 +728,8 @@ export class DeepScanCoordinator { } else if (!buffer.some((worker) => worker.id === outcome.worker.id)) { buffer.push(outcome.worker); } - removeValue(canceledWorkerIds, outcome.worker.id); - } else { - canceledWorkerIds.push(outcome.workerId); } } - this.audit.accepted = [...accepted]; - this.audit.omittedWorkerIds = unique(omittedWorkerIds); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); return firstFailure; }; const reconcileReducerSettlement = async (): Promise => { @@ -750,18 +743,11 @@ export class DeepScanCoordinator { const outcome = result.value; if ("status" in outcome) { buffer = [...outcome.consumed, ...buffer].sort(compareCompletionSequence); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); return outcome.error; } this.state = outcome.run; previousReducerResultPath = outcome.resultPath; - mergedDiscoveries.push(...outcome.consumed); - const { result: acceptedResult, ...metadata } = outcome; - latestResult = acceptedResult; - reducerOutcomes.push(metadata); - this.audit.reducers = [...reducerOutcomes]; - this.audit.mergedWorkerIds = unique(mergedDiscoveries.map((worker) => worker.id)); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); + latestResult = outcome.result; return undefined; }; @@ -809,7 +795,8 @@ export class DeepScanCoordinator { id: randomUUID(), label: `dedup-${String(reducerSequence).padStart(4, "0")}`, consumed, - previousReducerResultPath + previousReducerResultPath, + previousSourceCoverage: latestResult?.sourceCoverage, })); observe(reducer); } @@ -846,8 +833,6 @@ export class DeepScanCoordinator { const consecutiveErrors = outcome.consecutiveErrors ?? (this.state.consecutiveErrors ?? 0) + 1; this.state = { ...this.state, consecutiveErrors }; - canceledWorkerIds.push(outcome.workerId); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); this.log({ event: "discovery_worker_replaced", scanId: this.state.scanId, @@ -874,8 +859,6 @@ export class DeepScanCoordinator { throw outcome.error; } if (outcome.status === "canceled") { - canceledWorkerIds.push(outcome.workerId); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); if ( !this.abortController.signal.aborted && !this.discoveryAbortController.signal.aborted @@ -887,8 +870,6 @@ export class DeepScanCoordinator { accepted.push(outcome.worker); this.state = { ...this.state, consecutiveErrors: 0 }; buffer.push(outcome.worker); - this.audit.accepted = [...accepted]; - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); this.logProgress(accepted.length); continue; } @@ -896,7 +877,6 @@ export class DeepScanCoordinator { reducer = undefined; if ("status" in outcome) { buffer = [...outcome.consumed, ...buffer].sort(compareCompletionSequence); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); reducerFailures += 1; this.log({ event: "dedup_worker_replaced", @@ -919,22 +899,13 @@ export class DeepScanCoordinator { reducerFailures = 0; this.state = outcome.run; previousReducerResultPath = outcome.resultPath; - mergedDiscoveries.push(...outcome.consumed); - const { result: acceptedResult, ...metadata } = outcome; - latestResult = acceptedResult; - reducerOutcomes.push(metadata); - this.audit.reducers = [...reducerOutcomes]; - this.audit.mergedWorkerIds = unique(mergedDiscoveries.map((worker) => worker.id)); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); + latestResult = outcome.result; if ( !this.discoveryDeadlineReached && outcome.run.noNewStreak >= config.stopAfterNoNew && buffer.length === 0 ) { stopReason = "saturated"; - canceledWorkerIds.push(...active.keys()); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); - this.audit.bufferedWorkerIds = []; this.abortController.abort("deep_scan_saturated"); } } @@ -943,12 +914,6 @@ export class DeepScanCoordinator { // the manifest records which results completed and which were canceled. const lateFailure = await reconcileRemainingDiscoveries("omitted"); - this.audit.accepted = [...accepted]; - this.audit.mergedWorkerIds = unique(mergedDiscoveries.map((worker) => worker.id)); - this.audit.omittedWorkerIds = unique(omittedWorkerIds); - this.audit.canceledWorkerIds = unique(canceledWorkerIds); - this.audit.bufferedWorkerIds = buffer.map((worker) => worker.id); - // Once Deep reaches saturation, late worker errors cannot fail the scan. if (lateFailure && stopReason !== "saturated") throw lateFailure; @@ -961,11 +926,8 @@ export class DeepScanCoordinator { return { reason: stopReason ?? "capped", omittedWorkerIds: unique(omittedWorkerIds), - canceledWorkerIds: unique(canceledWorkerIds), - accepted, - mergedWorkerIds: unique(mergedDiscoveries.map((worker) => worker.id)), - reducers: reducerOutcomes, result: latestResult, + resultPath: previousReducerResultPath, }; } @@ -973,24 +935,29 @@ export class DeepScanCoordinator { const recovered: AcceptedDiscovery[] = []; for (const worker of this.state.persistedWorkers ?? []) { if (worker.kind !== "discovery" || worker.status !== "succeeded") continue; - if (!worker.resultManifestPath || !worker.completionSequence) { + // Migrated workers can have frozen merge inputs without an attempt record. + const claimedInput = this.state.persistedDedupInputs?.find((input) => ( + input.discoveryWorkerId === worker.id + && (input.attempt === undefined || input.attempt === worker.attempt) + && input.resultManifestPath + )); + const resultPath = worker.acceptedResultPath ?? claimedInput?.resultManifestPath ?? worker.resultManifestPath; + if (!resultPath || !worker.completionSequence) { throw new Error(`Accepted discovery ${worker.id} has incomplete persisted evidence.`); } await validateDiscoveryArtifacts( this.artifacts, - worker.resultManifestPath, + resultPath, this.state.scanId ); - const evidence = await persistedWorkerEvidence(worker); recovered.push({ id: worker.id, label: basename(dirname(worker.promptPath)), artifactDir: worker.artifactDir, - resultPath: worker.resultManifestPath, + resultPath, completionSequence: worker.completionSequence, attempt: worker.attempt, - ...(worker.threadId ? { threadId: worker.threadId } : {}), - ...evidence + ...(worker.threadId ? { threadId: worker.threadId } : {}) }); } return recovered.sort(compareCompletionSequence); @@ -998,87 +965,70 @@ export class DeepScanCoordinator { private async recoverCompletedReducers( discoveries: AcceptedDiscovery[] - ): Promise<{ reducers: AcceptedReducer[]; result?: DeepReductionInput }> { + ): Promise<{ result?: DeepReductionInput; resultPath?: string }> { const discoveriesById = new Map(discoveries.map((worker) => [worker.id, worker])); const inputs = this.state.persistedDedupInputs ?? []; - const outcomes: AcceptedReducer[] = []; let latestResult: DeepReductionInput | undefined; + let latestResultPath: string | undefined; const completedReducers = (this.state.persistedWorkers ?? []) .filter((worker) => worker.kind === "dedup" && worker.status === "succeeded") .sort((left, right) => ( workerLabelSequence(left, "dedup") - workerLabelSequence(right, "dedup") || left.id.localeCompare(right.id) )); - let noNewStreak = 0; for (const worker of completedReducers) { - if (!worker.resultManifestPath) { + // A later merge claim can retain a legacy aggregate's accepted reference. + const resultPath = worker.acceptedResultPath + ?? this.state.persistedMergeClaims?.find((claim) => ( + claim.previousWorkerId === worker.id && claim.previousResultPath + ))?.previousResultPath + ?? worker.resultManifestPath; + if (!resultPath) { throw new Error(`Completed reducer ${worker.id} has no persisted result manifest.`); } const consumed = inputs .filter((input) => input.dedupWorkerId === worker.id) .sort((left, right) => left.inputOrder - right.inputOrder) - .map((input) => discoveriesById.get(input.discoveryWorkerId)); + .map((input) => { + const discovery = discoveriesById.get(input.discoveryWorkerId); + return discovery && { + ...discovery, + resultPath: input.resultManifestPath ?? discovery.resultPath, + attempt: input.attempt ?? discovery.attempt, + }; + }); if (consumed.length === 0 || consumed.some((value) => !value)) { throw new Error(`Completed reducer ${worker.id} has incomplete persisted inputs.`); } const accepted = consumed as AcceptedDiscovery[]; - const { newFindings, result } = await validateReducerArtifacts({ + const claim = this.state.persistedMergeClaims?.find((item) => item.workerId === worker.id); + const { result } = await validateReducerArtifacts({ artifacts: this.artifacts, artifactDir: worker.artifactDir, - resultPath: worker.resultManifestPath, + resultPath, reducerId: worker.id, - previousReducerResultPath: outcomes.at(-1)?.resultPath + previousReducerResultPath: claim ? claim.previousResultPath : latestResultPath }, this.state.scanId); - latestResult = result; - noNewStreak = newFindings > 0 ? 0 : noNewStreak + accepted.length; - const evidence = await persistedWorkerEvidence(worker); - outcomes.push({ - type: "dedup", - id: worker.id, - consumed: accepted, - resultPath: worker.resultManifestPath, - newFindings, - attempt: worker.attempt, - ...(worker.threadId ? { threadId: worker.threadId } : {}), - ...evidence, - run: { ...this.state, noNewStreak } - }); - } - return { reducers: outcomes, result: latestResult }; - } - - private async recoverPersistedExecutions(): Promise { - const executions: WorkerExecutionAudit[] = []; - for (const worker of this.state.persistedWorkers ?? []) { - if ( - worker.kind === "setup" - || worker.status === "queued" - || worker.status === "running" - || (worker.status === "canceled" && worker.attempt === 0) - ) { - continue; + if (result.sourceCoverage === undefined) { + const context = { + root: worker.artifactDir, + repoRoot: this.state.targetPath, + scanId: this.state.scanId, + layout: "reducer" as const, + deepReducer: { + scanRoot: this.artifacts.scanDir, + claimedWorkers: accepted.map((source) => ({ + id: source.id, resultPath: source.resultPath, artifactDir: source.artifactDir, attempt: source.attempt, + })), + }, + }; + const sources = await readDeepReductionSources(context); + result.sourceCoverage = aggregateSourceCoverage(sources.discoveries, latestResult ?? null); } - const replaceableFailure = persistedReplaceableFailure(worker); - const status = replaceableFailure || worker.status === "failed" - ? "failed" - : worker.status; - executions.push({ - id: worker.id, - label: basename(dirname(worker.promptPath)), - kind: worker.kind, - status, - attempt: worker.attempt, - ...(worker.threadId ? { threadId: worker.threadId } : {}), - promptPath: worker.promptPath, - artifactDir: worker.artifactDir, - ...await persistedWorkerEvidence(worker), - ...(status === "failed" && worker.error - ? { error: replaceableFailure?.message ?? worker.error } - : {}), - ...(replaceableFailure ? { failureKind: replaceableFailure.kind } : {}) - }); + latestResult = result; + latestResultPath = resultPath; } - return executions; + return { result: latestResult, resultPath: latestResultPath }; } private reducerReady( @@ -1119,36 +1069,7 @@ export class DeepScanCoordinator { }); } - /** - * A workbench process can commit SQLite and still lose its stdout response. - * Replay the exact idempotent finish once before treating the run as failed; - * otherwise we could overwrite a successful terminal state after durable success. - */ - private async finishWithReplay(result: SchedulerResult): Promise { - const input = { - scanId: this.state.scanId, - reason: result.reason, - manifestPath: join(this.state.scanDir, "scan-manifest.json"), - omittedWorkerIds: result.omittedWorkerIds - }; - try { - return await this.options.store.finish(input); - } catch (firstError) { - this.log({ - event: "coordinator_finish_replay", - scanId: this.state.scanId, - reason: errorKind(firstError) - }); - try { - return await this.options.store.finish(input); - } catch (replayError) { - throw new Error( - `Deep Scan terminal persistence replay failed: ${errorMessage(replayError)}`, - { cause: firstError } - ); - } - } - } + } const systemClock: DeepScanClock = { @@ -1186,12 +1107,6 @@ function unique(values: string[]): string[] { return [...new Set(values)]; } -function removeValue(values: string[], value: string): void { - for (let index = values.length - 1; index >= 0; index -= 1) { - if (values[index] === value) values.splice(index, 1); - } -} - function cloneState(state: DeepScanRunState): DeepScanRunState { return { ...state, config: { ...state.config } }; } @@ -1238,30 +1153,6 @@ function persistedReplaceableFailure( return undefined; } -async function persistedWorkerEvidence(worker: PersistedDeepScanWorker): Promise<{ - basePromptSha256: string; - attemptPromptPaths: string[]; -}> { - const attemptPromptPaths = [worker.promptPath]; - for (let attempt = 2; attempt <= worker.attempt; attempt += 1) { - const promptPath = join( - dirname(worker.promptPath), - "prompts", - `attempt-${String(attempt).padStart(2, "0")}.md` - ); - try { - await fs.access(promptPath); - attemptPromptPaths.push(promptPath); - } catch { - // Transient execution retries reuse the original prompt. - } - } - return { - basePromptSha256: sha256(await fs.readFile(worker.promptPath, "utf8")), - attemptPromptPaths - }; -} - function discoveryErrorLimitError( count: number, limit: number, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts index 788745ae4..51db8dedd 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts @@ -1,7 +1,11 @@ import { accessSync, constants as fsConstants, existsSync, promises as fs, readdirSync, statSync } from "node:fs"; import { createRequire } from "node:module"; import { delimiter, dirname, isAbsolute, join, resolve, win32 } from "node:path"; -import { Codex } from "@openai/codex-sdk"; +import { + createCodexClient, + readCodexSessionTurn +} from "../../../../../sdk/typescript/src/codex-session.js"; +import type { CodexOptions } from "@openai/codex-sdk"; import { parse as parseToml } from "smol-toml"; import { executablePathForSpawn } from "./executable-path.js"; import { @@ -22,6 +26,8 @@ import type { } from "./types.js"; export interface CodexSdkWorkerModelSettings { + /** Resolved by the execution owner, including when reconstructing a scan. */ + codexOptions?: CodexOptions; model?: string; reasoningEffort?: string; artifactContext?: CodexSdkWorkerArtifactContext; @@ -39,7 +45,7 @@ export interface CodexSdkWorkerArtifactContext { } export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { - private runtimeReasoningSummary?: Promise; + private runtimeModelConfig?: Promise>; constructor(private readonly modelSettings: CodexSdkWorkerModelSettings = {}) {} @@ -52,15 +58,32 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { ); } const workerProfile = workerPermissionProfile(parentSandbox); - const configOverrides = workerPermissionProfileConfigOverrides(workerProfile); + const resolved = this.modelSettings.codexOptions; const originalCwd = process.cwd(); - const childEnv = await snapshotWorkerEnvironment(); - // Snapshot the SDK's per-scan config once for this coordinator, including resumes. - const reasoningSummary = await (this.runtimeReasoningSummary ??= workerReasoningSummary(childEnv)); + const childEnv = await snapshotWorkerEnvironment(resolved?.env); + if (resolved?.apiKey !== undefined) childEnv.CODEX_API_KEY = resolved.apiKey; + // Snapshot per-scan selections once; a reconstructed owner can supply them. + // Native account credentials continue to refresh in the selected home. + const modelConfig: NonNullable = { + ...await (this.runtimeModelConfig ??= resolved?.config + ? Promise.resolve(resolved.config) + : workerModelConfig(childEnv)), + ...(this.modelSettings.model ? { model: this.modelSettings.model } : {}), + // The CLI can add effort levels before the pinned SDK widens ThreadOptions. + ...(this.modelSettings.reasoningEffort + ? { model_reasoning_effort: this.modelSettings.reasoningEffort } + : {}) + }; + const configOverrides = [ + ...(resolved?.configOverrides ?? []), + ...workerPermissionProfileConfigOverrides(workerProfile) + ]; const openAiApiKey = environmentVariable(childEnv, "OPENAI_API_KEY", process.platform)?.trim(); const codexApiKey = environmentVariable(childEnv, "CODEX_API_KEY", process.platform)?.trim(); const codexPath = resolveCodexPath( - childEnv, + resolved?.codexPathOverride === undefined + ? childEnv + : { ...childEnv, CODEX_CLI_PATH: resolved.codexPathOverride }, process.platform, process.arch, originalCwd @@ -69,34 +92,35 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { codexPath, cwd: request.workingDirectory, profileId: DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID, - configOverrides, + configOverrides: [ + ...Object.entries(workerModelSelection(modelConfig)) + .map(([key, value]) => `${key}=${tomlInlineValue(value)}`), + ...configOverrides, + ...(resolved?.baseUrl ? [`openai_base_url=${tomlString(resolved.baseUrl)}`] : []) + ], expectedProfile: workerProfile, env: childEnv, allowOpenAiApiKeyFallback: Boolean(openAiApiKey && !codexApiKey), signal: request.signal }); const prompt = await fs.readFile(request.promptPath, "utf8"); - const codex = new Codex({ + const codex = createCodexClient({ + ...resolved, codexPathOverride: executablePathForSpawn(codexPath), env: childEnv, // Codex exec reads CODEX_API_KEY; the SDK maps apiKey to that variable. // Keep native credentials unless the worker has no configured account. ...(useOpenAiApiKey ? { apiKey: openAiApiKey } : {}), config: { - ...(reasoningSummary === undefined - ? {} - : { model_reasoning_summary: reasoningSummary }), - // The CLI can add effort levels before the pinned SDK widens ThreadOptions. - ...(this.modelSettings.reasoningEffort - ? { model_reasoning_effort: this.modelSettings.reasoningEffort } - : {}), + ...modelConfig, mcp_servers: { + ...(isRecord(modelConfig.mcp_servers) ? modelConfig.mcp_servers : {}), // Discovery workers use the bundled skills and artifacts, not the parent workbench MCP. // A disabled server still needs a valid transport while Codex resolves plugin configuration. "codex-security": { command: "node", enabled: false }, ...this.compactArtifactServer(request) }, - ...workerSubagentConfig(request.subagents) + ...workerSubagentConfig(request.subagents, modelConfig) }, // Structured SDK config cannot preserve literal filesystem keys such as // ":root" or "/repo/.env"; raw overrides keep this inline TOML intact. @@ -110,7 +134,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { workingDirectory: request.workingDirectory } as const; const thread = request.resumeThreadId - ? codex.resumeThread(request.resumeThreadId, threadOptions) + ? codex.resumeThread!(request.resumeThreadId, threadOptions) : codex.startThread(threadOptions); const input = request.resumeThreadId ? request.continuationPrompt ?? prompt @@ -125,51 +149,44 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { try { const { events } = await thread.runStreamed(input, { signal: controller.signal }); - let finalResponse = ""; - let threadId: string | undefined; - let turnCompleted = false; - let lastStreamError: string | undefined; const diagnostics: CodexWorkerDiagnostic[] = []; - for await (const event of events) { - if (event.type === "thread.started") { - threadId = event.thread_id; - await request.onThreadStarted?.(threadId); - } else if (event.type === "item.completed") { - const fallbackError = event.item.type === "error" - ? deepScanPermissionProfileFallbackError(event.item.message) - : undefined; - if (fallbackError) { - controller.abort(fallbackError); - throw fallbackError; - } - if (event.item.type === "agent_message") { - finalResponse = event.item.text; - } else { + const turn = await readCodexSessionTurn({ + thread, + events, + stopOnCompletion: true, + onEvent: async (event) => { + if (event.type === "thread.started" && typeof event.thread_id === "string") { + await request.onThreadStarted?.(event.thread_id); + } else if (event.type === "item.completed" && isRecord(event.item)) { + const fallbackError = event.item.type === "error" && typeof event.item.message === "string" + ? deepScanPermissionProfileFallbackError(event.item.message) + : undefined; + if (fallbackError) { + controller.abort(fallbackError); + throw fallbackError; + } appendSafeItemDiagnostic(diagnostics, event.item); + } else if (event.type === "turn.completed") { + request.signal.removeEventListener("abort", forwardAbort); + } else if (event.type === "turn.failed") { + throw new Error((event.error as { message: string }).message); + } else if (event.type === "error" && typeof event.message === "string") { + const fallbackError = deepScanPermissionProfileFallbackError(event.message); + if (fallbackError) { + controller.abort(fallbackError); + throw fallbackError; + } + // Codex exec emits retry-in-progress notifications as error events. } - } else if (event.type === "turn.completed") { - turnCompleted = true; - request.signal.removeEventListener("abort", forwardAbort); - break; - } else if (event.type === "turn.failed") { - throw new Error(event.error.message); - } else if (event.type === "error") { - const fallbackError = deepScanPermissionProfileFallbackError(event.message); - if (fallbackError) { - controller.abort(fallbackError); - throw fallbackError; - } - // Codex exec currently emits retry-in-progress notifications as error events. - lastStreamError = event.message; } - } - if (!turnCompleted) { - const detail = lastStreamError ? `: ${lastStreamError}` : ""; + }); + if (turn.status !== "completed") { + const detail = turn.lastStreamError ? `: ${turn.lastStreamError}` : ""; throw new Error(`Codex worker stream ended before turn.completed${detail}`); } return { - finalResponse, - threadId: threadId ?? thread.id ?? undefined, + finalResponse: turn.finalResponse, + threadId: turn.threadId ?? thread.id ?? undefined, ...(diagnostics.length > 0 ? { diagnostics } : {}) }; } finally { @@ -246,13 +263,16 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { } } -function workerSubagentConfig(subagents: number) { +function workerSubagentConfig(subagents: number, config: NonNullable) { return { // V1 counts children; V2 counts the root plus its children. Keeping its // feature disabled lets the model choose either runtime without rejecting // inherited agents.max_threads configuration. - ...(subagents > 0 ? { agents: { max_threads: subagents } } : {}), + ...(subagents > 0 + ? { agents: { ...(isRecord(config.agents) ? config.agents : {}), max_threads: subagents } } + : {}), features: { + ...(isRecord(config.features) ? config.features : {}), multi_agent_v2: { enabled: false, max_concurrent_threads_per_session: subagents + 1 @@ -268,7 +288,7 @@ function workerSubagentConfig(subagents: number) { }; } -type TomlValue = string | number | boolean | TomlObject; +type TomlValue = string | number | boolean | TomlValue[] | TomlObject; type TomlObject = { [key: string]: TomlValue }; function workerPermissionProfile( @@ -307,6 +327,7 @@ function tomlInlineValue(value: TomlValue): string { if (typeof value === "string") return tomlString(value); if (typeof value === "number") return String(value); if (typeof value === "boolean") return value ? "true" : "false"; + if (Array.isArray(value)) return `[${value.map(tomlInlineValue).join(",")}]`; return `{${Object.entries(value) .map(([key, entry]) => `${tomlKey(key)}=${tomlInlineValue(entry)}`) .join(",")}}`; @@ -383,30 +404,38 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } -async function workerReasoningSummary(environment: Record): Promise { +// These are the existing non-secret selections written by the SDK preflight +// adapter. Reading only summary left provider selection in a shared home. +function workerModelSelection(config: NonNullable): TomlObject { + const result: TomlObject = {}; + for (const key of ["model", "model_provider", "model_reasoning_effort", "model_reasoning_summary", "service_tier", "model_providers"]) { + const value = config[key]; + if (value !== undefined) result[key] = value; + } + return result; +} + +async function workerModelConfig(environment: Record): Promise> { const configPath = environmentVariable(environment, "CODEX_SECURITY_CONFIG_PATH", process.platform); - if (!configPath) return undefined; + if (!configPath) return {}; const config = parseToml(await fs.readFile(configPath, "utf8")); const profiles = config.profiles; const profile = typeof config.profile === "string" && isRecord(profiles) ? profiles[config.profile] : undefined; - const summary = isRecord(profile) && profile.model_reasoning_summary !== undefined - ? profile.model_reasoning_summary - : config.model_reasoning_summary; - return typeof summary === "string" ? summary : undefined; + return workerModelSelection({ ...config, ...(isRecord(profile) ? profile : {}) } as NonNullable); } -async function snapshotWorkerEnvironment(): Promise> { +async function snapshotWorkerEnvironment(source: NodeJS.ProcessEnv = process.env): Promise> { const environment = Object.fromEntries( - Object.entries(process.env) + Object.entries(source) .filter((entry): entry is [string, string] => entry[1] !== undefined) ) as Record; if (process.platform === "win32") { // process.env is case-insensitive on Windows; a plain object is not. // Keep its selected values while giving the child one spelling per key. for (const name of ["CODEX_CLI_PATH", "CODEX_HOME", "CODEX_MANAGED_PACKAGE_ROOT", "LOCALAPPDATA"]) { - const value = process.env[name]; + const value = environmentVariable(source, name, process.platform); for (const key of Object.keys(environment)) { if (key.toUpperCase() === name) delete environment[key]; } diff --git a/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts new file mode 100644 index 000000000..8e2fb6f4b --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/finalization.ts @@ -0,0 +1,183 @@ +import { + deepReductionToScanDraft, + parseDeepReduction, +} from "./artifact-validation.js"; +import { + createScanArtifactContext, + type RunArtifactWorkbench, +} from "../artifact-context.js"; +import { + recordCodexSecurityScanDraftViaWorkbench, + type DeepScanPublication, +} from "../artifact-scan-draft.js"; +import { WorkbenchDeepScanStore } from "./store.js"; +import { createDeepScanArtifacts } from "./artifacts.js"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { type ScanDraftInput } from "../artifact-scan-draft.js"; +import { requireRegularFile, type DeepScanArtifacts } from "./artifacts.js"; +import type { + DeepScanFinalizationInput, + DeepScanRunState, + DeepScanStore, +} from "./types.js"; + +export type { DeepScanFinalizationInput } from "./types.js"; + +/** Publish exactly the saved aggregate; the enclosing scan owns public completion. */ +export async function publishSelectedDeepScan(input: { + run: DeepScanRunState; + artifacts: DeepScanArtifacts; + signal: AbortSignal; + publish: ( + draft: ScanDraftInput, + signal: AbortSignal, + publication: DeepScanPublication, + ) => Promise; + finish: DeepScanStore["finish"]; +}): Promise { + const { run, artifacts, signal } = input; + const selection = selectedInput(run); + if (run.status === "succeeded") return run; + if (run.status !== "running") + throw new Error("Stopped Deep Scan finalization cannot become successful."); + signal.throwIfAborted(); + const draft = await readSelectedDeepScanDraft( + artifacts, + run.scanId, + selection, + ); + await input.publish(draft, signal, { + coordinatorGeneration: run.coordinatorGeneration, + resultPath: + selection.resultPath === null + ? null + : join(run.scanDir, selection.resultPath), + }); + signal.throwIfAborted(); + return input.finish({ + scanId: run.scanId, + reason: selection.terminalReason, + manifestPath: join(run.scanDir, "scan-manifest.json"), + omittedWorkerIds: selection.omittedWorkerIds, + }); +} + +/** Recreate the chosen draft without scheduling discovery or reducer work. */ +export async function readSelectedDeepScanDraft( + artifacts: DeepScanArtifacts, + scanId: string, + selection: DeepScanFinalizationInput, +): Promise { + if (selection.version !== 1) + throw new Error("Unsupported Deep Scan finalization input version."); + if (selection.resultPath === null) { + if ( + selection.terminalReason !== "capped" || + selection.resultSha256 !== null + ) { + throw new Error( + "An empty Deep Scan finalization requires the recorded discovery deadline.", + ); + } + return { + scanId, + findings: [], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { + reason: + "The configured discovery time limit elapsed before any source review completed.", + }, + ], + }, + }; + } + const resultPath = join(artifacts.scanDir, selection.resultPath); + await requireRegularFile(resultPath, artifacts.scanDir); + const contents = await readFile(resultPath); + if ( + createHash("sha256").update(contents).digest("hex") !== + selection.resultSha256 + ) { + throw new Error( + "The selected Deep Scan finalization input changed after acceptance.", + ); + } + const stored = JSON.parse(contents.toString("utf8")); + const draft = deepReductionToScanDraft(parseDeepReduction(stored, true)); + if (draft.scanId !== scanId || draft.complete === false) { + throw new Error( + "Deep Scan finalization requires the selected complete result for this scan.", + ); + } + return draft; +} + +/** SDK recovery uses the installed plugin's publisher and the original parent scan. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: RunArtifactWorkbench; + signal: AbortSignal; + handoffClaimToken?: string; +}): Promise { + const store = new WorkbenchDeepScanStore(input.runWorkbench); + const run = await store.get(input.scanId, input.threadId); + selectedInput(run); + try { + await publishSelectedDeepScan({ + run, + artifacts: createDeepScanArtifacts(run.scanDir), + signal: input.signal, + publish: async (draft, signal, publication) => { + const context = await createScanArtifactContext( + input.scanId, + input.runWorkbench, + { + requireRunning: true, + requireClaim: true, + handoffClaimToken: input.handoffClaimToken, + pluginRoot: input.pluginRoot, + }, + ); + await recordCodexSecurityScanDraftViaWorkbench( + context, + draft, + input.runWorkbench, + signal, + publication, + ); + }, + finish: (selection) => + store.finish({ + ...selection, + coordinatorGeneration: run.coordinatorGeneration, + }), + }); + } catch (error) { + // The original coordinator may publish while the SDK recovers its parent turn. + const committed = await store + .get(input.scanId, input.threadId) + .catch(() => null); + if (committed?.status !== "succeeded") throw error; + } +} + +function selectedInput(run: DeepScanRunState): DeepScanFinalizationInput { + const selection = run.finalizationInput; + if (!selection) + throw new Error("Deep Scan has no selected finalization input."); + if ( + run.workflowVersion !== "deep-security-scan/v2" || + selection.version !== 1 + ) { + throw new Error("Unsupported Deep Scan finalization input version."); + } + return selection; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts new file mode 100644 index 000000000..2f07eb8e9 --- /dev/null +++ b/plugins/codex-security/mcp-app/src/deep-scan/recovery-settings.ts @@ -0,0 +1,279 @@ +import { promises as fs } from "node:fs"; +import { homedir } from "node:os"; +import { isAbsolute, join, win32 } from "node:path"; +import type { CodexOptions } from "@openai/codex-sdk"; +import { parse as parseToml } from "smol-toml"; +import { scanPreflightCodexConfig } from "../../../../../sdk/typescript/src/preflight-config.js"; +import { resolveCodexProfile, type JsonObject } from "../../../../../sdk/typescript/src/config.js"; +import { readScanLogs } from "../../../../../sdk/typescript/src/scan-logs.js"; +import { resolveCodexPath } from "./executor.js"; +import type { DeepWorkerParentSandbox } from "./parent-sandbox.js"; +import type { DeepScanRunState } from "./types.js"; + +/** Credentials and arbitrary environment/configuration stay with Codex. */ +export interface DeepScanExecutionSettings { + codexPath: string; + codexHome: string; + model?: string; + modelProvider?: string; + reasoningEffort?: string; + reasoningSummary?: string; + serviceTier?: string; + /** The native snapshot recorded no request tier; serviceTier preserves its wire behavior. */ + nativeServiceTierAbsent?: true; + providerConfig?: JsonObject; + parentSandbox?: DeepWorkerParentSandbox; +} + +export interface DeepScanLegacySettingsContext { + config?: JsonObject; + usageOwner?: DeepScanRunState["usageOwner"]; +} + +export async function captureDeepScanExecutionSettings( + original: Pick, + parentSandbox: DeepWorkerParentSandbox, + environment: NodeJS.ProcessEnv = process.env, + parent?: { threadId: string; startedAt?: string } +): Promise { + const codexHome = environment.CODEX_HOME || join(homedir(), ".codex"); + const configPath = environment.CODEX_SECURITY_CONFIG_PATH ?? join(codexHome, "config.toml"); + let config: JsonObject; + try { + config = parseToml(await fs.readFile(configPath, "utf8")) as JsonObject; + } catch (error) { + if (environment.CODEX_SECURITY_CONFIG_PATH || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + config = {}; + } + // Reuse the SDK projection: custom provider credentials belong in the native home. + const selected = scanPreflightCodexConfig(resolveCodexProfile(config)); + // A recovered scan can have a different continuation. Only its recorded owner + // establishes original history; null means that historical binding is missing. + const owner = original.usageOwner === undefined ? parent : original.usageOwner; + const native = !owner?.threadId ? {} : await originalParentSettings(codexHome, { + ...owner, threadId: owner.threadId, startedAt: parent?.startedAt ?? owner.startedAt + }); + return executionSettings({ + codexPath: resolveCodexPath(environment, process.platform, process.arch, process.cwd()), + codexHome: !isAbsolute(codexHome) + || (process.platform === "win32" && ["\\", "/"].includes(win32.parse(codexHome).root)) + ? await fs.realpath(codexHome) : codexHome, + model: original.model ?? (selected.model as string | undefined) ?? native.model, + reasoningEffort: original.reasoningEffort ?? (selected.model_reasoning_effort as string | undefined) ?? native.reasoningEffort, + modelProvider: (selected.model_provider as string | undefined) ?? native.modelProvider, + reasoningSummary: (selected.model_reasoning_summary as string | undefined) ?? native.reasoningSummary, + serviceTier: (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } : {}), + providerConfig: selected.model_providers as JsonObject | undefined, + parentSandbox + }); +} + +async function originalParentSettings( + codexHome: string, + parent: { threadId: string; turnId?: string | null; startedAt?: string } +): Promise> { + // Native config/read represents omitted selections as null. Recover recorded + // selections from the original parent; some native records omit the summary. + // History can be disabled or unavailable; configured selections still work. + try { + const log = await readScanLogs({ + scanId: parent.threadId, threadId: parent.threadId, executionThreadIds: [], + codexHome, allowMissingRoot: true + }); + const settings: Partial = {}; + let applied: Partial | undefined; + let summaryIsCompatibilityOnly = false; + const cutoff = parent.startedAt === undefined ? Infinity : Date.parse(parent.startedAt); + for (const entry of log.events) { + const event = entry.event as Record; + const timestamp = typeof event.timestamp === "string" ? Date.parse(event.timestamp) : undefined; + if (timestamp !== undefined && timestamp > cutoff) continue; + const payload = event.payload; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) continue; + const context = payload as Record; + if (event.type === "event_msg" && context.type === "thread_settings_applied") { + if (typeof context.thread_id === "string" && context.thread_id !== parent.threadId) continue; + const snapshot = context.thread_settings; + if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) continue; + const value = snapshot as Record; + applied = { + model: typeof value.model === "string" ? value.model : undefined, + modelProvider: typeof value.model_provider_id === "string" ? value.model_provider_id : undefined, + reasoningEffort: typeof value.reasoning_effort === "string" ? value.reasoning_effort : undefined, + reasoningSummary: typeof value.reasoning_summary === "string" ? value.reasoning_summary : undefined, + // A persisted native absent tier and explicit standard both omit the + // request tier. This does not infer a tier from missing history. + serviceTier: typeof value.service_tier === "string" ? value.service_tier + : value.service_tier === undefined ? "default" : undefined, + ...(value.service_tier === undefined ? { nativeServiceTierAbsent: true as const } : {}) + }; + } + if (event.type === "session_meta") { + if (typeof context.model_provider === "string") settings.modelProvider = context.model_provider; + // Codex 0.133 replaced turn_context.summary with a compatibility default. + // Fresh threads need not have a thread_settings_applied record to replace it. + const version = typeof context.cli_version === "string" + ? /^(\d+)\.(\d+)\./u.exec(context.cli_version) : null; + summaryIsCompatibilityOnly = version !== null + && (Number(version[1]) > 0 || Number(version[2]) >= 133); + } + if (event.type === "turn_context") { + if (parent.turnId && context.turn_id !== parent.turnId) continue; + if (typeof context.model === "string") settings.model = context.model; + if (typeof context.effort === "string") settings.reasoningEffort = context.effort; + if (!summaryIsCompatibilityOnly && typeof context.summary === "string") settings.reasoningSummary = context.summary; + } + } + // Applied snapshots contain native selected values. Newer turn-context + // summaries are only a compatibility field, not the active selection. + return { ...settings, ...applied }; + } catch { + return {}; + } +} + +/** New runs save settings in their creation transaction, before any coordinator claim. */ +export async function loadDeepScanExecutionSettings( + scanDir: string, + original?: Pick, + readLegacyContext?: () => Promise, + environment: NodeJS.ProcessEnv = process.env +): Promise> { + const path = join(scanDir, "artifacts", "deep_discovery", "execution-settings.json"); + let settings: DeepScanExecutionSettings; + try { + const saved = JSON.parse(await fs.readFile(path, "utf8")); + if (saved.version !== 1) { + throw new Error("This Deep Scan uses an unsupported execution settings version."); + } + settings = executionSettings(saved.settings); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (original?.workflowVersion === "deep-security-scan/v1" || original?.workflowVersion === "deep-scan-mcp/v1") { + // Legacy runs predate this file. Their saved recipe and recorded owner + // can recover selections, but cannot establish an original executable or + // home. Leave those unknown and retain the existing native launch behavior. + const context = await readLegacyContext?.(); + const selected = scanPreflightCodexConfig(resolveCodexProfile(context?.config ?? {})); + const owner = original.usageOwner ?? context?.usageOwner; + const home = environment.CODEX_HOME || join(homedir(), ".codex"); + const native = !owner?.threadId ? {} : await originalParentSettings(home, { + ...owner, threadId: owner.threadId, startedAt: original.createdAt ?? owner.startedAt + }); + return { + model: original.model ?? (selected.model as string | undefined) ?? native.model, + reasoningEffort: original.reasoningEffort ?? (selected.model_reasoning_effort as string | undefined) ?? native.reasoningEffort, + modelProvider: (selected.model_provider as string | undefined) ?? native.modelProvider, + reasoningSummary: (selected.model_reasoning_summary as string | undefined) ?? native.reasoningSummary, + serviceTier: (selected.service_tier as string | undefined) ?? native.serviceTier, + ...(selected.service_tier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } : {}), + providerConfig: selected.model_provider === "amazon-bedrock" + ? selected.model_providers as JsonObject | undefined : undefined + }; + } + throw new Error("This Deep Scan has no recorded original execution settings; its executable and Codex home cannot be recovered."); + } + if (!original || (settings.model !== undefined && settings.reasoningEffort !== undefined + && settings.modelProvider !== undefined && settings.reasoningSummary !== undefined + && settings.serviceTier !== undefined)) return settings; + // Earlier snapshots can omit native selections. Recover only from the saved + // home and recorded owner; the continuation's current config is not history. + const owner = original.usageOwner; + const native = !owner?.threadId ? {} : await originalParentSettings(settings.codexHome, { + ...owner, threadId: owner.threadId, startedAt: original.createdAt + }); + // History reads can outlive this coordinator. Project missing selections for + // its workers without overwriting a snapshot owned by a newer coordinator. + return executionSettings({ + ...settings, + model: settings.model ?? original.model ?? native.model, + reasoningEffort: settings.reasoningEffort ?? original.reasoningEffort ?? native.reasoningEffort, + modelProvider: settings.modelProvider ?? native.modelProvider, + reasoningSummary: settings.reasoningSummary ?? native.reasoningSummary, + serviceTier: settings.serviceTier ?? native.serviceTier, + ...(settings.serviceTier === undefined && native.nativeServiceTierAbsent + ? { nativeServiceTierAbsent: true as const } : {}) + }); +} + +export function restoredDeepScanWorkerSettings( + settings: Partial, + currentParentSandbox: DeepWorkerParentSandbox, + environment: () => NodeJS.ProcessEnv = () => process.env +): { + codexOptions: CodexOptions; + model?: string; + reasoningEffort?: string; + parentSandbox: DeepWorkerParentSandbox; +} { + const originalSandbox = settings.parentSandbox; + const depths = [originalSandbox?.globScanMaxDepth, currentParentSandbox.globScanMaxDepth] + .filter((depth): depth is number => depth !== undefined); + // Native depth caps limit deny-glob expansion, not allowed traversal. Keep + // the larger finite cap, or no cap when either known policy has uncapped globs. + const uncapped = [originalSandbox, currentParentSandbox].some((sandbox) => + sandbox?.globScanMaxDepth === undefined && sandbox?.filesystemDenies.some((path) => + ["*", "?", "[", "]"].some((character) => path.includes(character)))); + return { + model: settings.model, + reasoningEffort: settings.reasoningEffort, + parentSandbox: { + filesystemDenies: [...new Set([ + ...(originalSandbox?.filesystemDenies ?? []), ...currentParentSandbox.filesystemDenies + ])], + ...(uncapped || depths.length === 0 ? {} : { globScanMaxDepth: Math.max(...depths) }) + }, + codexOptions: { + codexPathOverride: settings.codexPath, + // The executor reads this property for each launch. API keys can refresh; + // only the original account home and non-secret selections are bound. + get env() { + return Object.fromEntries(Object.entries({ ...environment(), + ...(settings.codexPath === undefined ? {} : { CODEX_CLI_PATH: settings.codexPath }), + ...(settings.codexHome === undefined ? {} : { CODEX_HOME: settings.codexHome }) }) + .filter((entry): entry is [string, string] => entry[1] !== undefined)); + }, + config: scanPreflightCodexConfig({ + ...(settings.model === undefined ? {} : { model: settings.model }), + ...(settings.reasoningEffort === undefined ? {} : { model_reasoning_effort: settings.reasoningEffort }), + ...(settings.modelProvider === undefined ? {} : { model_provider: settings.modelProvider }), + ...(settings.reasoningSummary === undefined ? {} : { model_reasoning_summary: settings.reasoningSummary }), + ...(settings.serviceTier === undefined ? {} : { service_tier: settings.serviceTier }), + ...(settings.providerConfig === undefined ? {} : { model_providers: settings.providerConfig }) + }) as NonNullable + } + }; +} + +function executionSettings(value: DeepScanExecutionSettings): DeepScanExecutionSettings { + // Catalog provider definitions are reconstructed by the existing launch + // projection. Only Bedrock's per-scan AWS selectors need persistence. + const provider = value.modelProvider === "amazon-bedrock" ? scanPreflightCodexConfig({ + ...(value.modelProvider === undefined ? {} : { model_provider: value.modelProvider }), + ...(value.providerConfig === undefined ? {} : { model_providers: value.providerConfig }) + }).model_providers as JsonObject | undefined : undefined; + const settings: DeepScanExecutionSettings = { + codexPath: value.codexPath, + codexHome: value.codexHome, + model: value.model, + modelProvider: value.modelProvider, + reasoningEffort: value.reasoningEffort, + reasoningSummary: value.reasoningSummary, + serviceTier: value.serviceTier, + ...(value.nativeServiceTierAbsent === true ? { nativeServiceTierAbsent: true } : {}), + ...(provider === undefined ? {} : { providerConfig: provider }), + ...(value.parentSandbox === undefined ? {} : { parentSandbox: { + filesystemDenies: [...value.parentSandbox.filesystemDenies], + ...(value.parentSandbox.globScanMaxDepth === undefined ? {} : { + globScanMaxDepth: value.parentSandbox.globScanMaxDepth + }) + } }) + }; + if (typeof settings.codexPath !== "string" || typeof settings.codexHome !== "string") { + throw new Error("Deep Scan execution settings are missing the recorded executable or Codex home."); + } + return settings; +} diff --git a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts index 574ff2c53..3bbac0a2a 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/registry.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/registry.ts @@ -2,13 +2,18 @@ import { setTimeout as delay } from "node:timers/promises"; import { DeepScanCoordinator } from "./coordinator.js"; import type { CoordinatorOptions } from "./coordinator.js"; import { isTransientPersistenceError } from "./store.js"; -import type { BeginDeepScanResult, DeepScanCoordinatorClaim, DeepScanRunState } from "./types.js"; +import type { BeginDeepScanResult, CodexWorkerExecutor, DeepScanCoordinatorClaim, DeepScanRunState } from "./types.js"; const COORDINATOR_LEASE_MS = 30_000; const COORDINATOR_POLL_MS = 1_000; export { DeepScanCoordinator, DeepScanNonRetryableError } from "./coordinator.js"; +export interface DeepScanCoordinatorStartOptions extends CoordinatorOptions { + /** Resolve persisted execution settings only after acquiring this run. */ + prepareExecutor?: (run: DeepScanRunState) => Promise; +} + /** Owns the live coordinators in this MCP server process. */ export class DeepScanCoordinatorRegistry { private readonly coordinators = new Map(); @@ -17,7 +22,8 @@ export class DeepScanCoordinatorRegistry { return this.coordinators.get(scanId); } - start(options: CoordinatorOptions): DeepScanCoordinator { + start(options: DeepScanCoordinatorStartOptions): DeepScanCoordinator { + requireSupportedDeepScan(options.run); const existing = this.coordinators.get(options.run.scanId); if (existing) return existing; const { observeReplacement: _unused, ...remoteOptions } = options; @@ -104,7 +110,7 @@ export class DeepScanRemoteCoordinator { private readonly input: { run: DeepScanRunState; registry: Pick; - options: Omit; + options: Omit; } ) {} @@ -136,6 +142,7 @@ export class DeepScanRemoteCoordinator { continue; } if (run.status !== "running") return run; + requireSupportedDeepScan(run); const heartbeat = run.updatedAt ? Date.parse(run.updatedAt) : Number.NaN; if ( @@ -167,7 +174,7 @@ export class DeepScanRemoteCoordinator { } } if (claim?.acquired) { - const coordinator = registry.start({ ...options, run: claim.run }); + const coordinator = await startClaimedCoordinator(registry, options, claim.run); return deadline === undefined ? await coordinator.wait(signal) : await coordinator.wait(signal, Math.max(0, deadline - Date.now())); @@ -185,11 +192,12 @@ export class DeepScanRemoteCoordinator { export async function startOrJoinDeepScanCoordinator(input: { begin: BeginDeepScanResult; registry: Pick; - options: Omit; + options: Omit; }): Promise<{ coordinator: DeepScanCoordinator | DeepScanRemoteCoordinator; joined: boolean; }> { + requireSupportedDeepScan(input.begin.run); const existing = input.registry.get(input.begin.run.scanId); if (existing) return { coordinator: existing, joined: true }; const threadId = input.options.threadId; @@ -212,11 +220,44 @@ export async function startOrJoinDeepScanCoordinator(input: { }; } return { - coordinator: input.registry.start({ ...input.options, run: claim.run }), + coordinator: await startClaimedCoordinator(input.registry, input.options, claim.run), joined: false }; } +async function startClaimedCoordinator( + registry: Pick, + options: Omit, + run: DeepScanRunState +): Promise { + requireSupportedDeepScan(run); + const executor = options.prepareExecutor && !run.finalizationInput + ? await options.prepareExecutor(run) + : options.executor; + return registry.start({ ...options, executor, run }); +} + +function requireSupportedDeepScan(run: DeepScanRunState): void { + if (run.finalizationInput !== undefined && ( + run.workflowVersion !== "deep-security-scan/v2" || run.finalizationInput.version !== 1 + )) { + throw new Error("This executor does not support this Deep Scan finalization input version."); + } + // Missing versions are supported for older adapters that did not project them. + if ( + (run.schemaVersion !== undefined && run.schemaVersion !== 1) + || (run.workflowVersion !== undefined + && run.workflowVersion !== "deep-security-scan/v1" + && run.workflowVersion !== "deep-scan-mcp/v1" + && run.workflowVersion !== "deep-security-scan/v2") + ) { + throw new Error( + "This Deep Scan uses an unsupported workflow or schema version. " + + "Resume it with a compatible Codex Security release." + ); + } +} + function remoteAbortError(reason: unknown): Error { const error = new Error("Deep Scan observation was aborted.", { cause: reason }); error.name = "AbortError"; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/store.ts b/plugins/codex-security/mcp-app/src/deep-scan/store.ts index 8bf6b0bf7..106aa4e32 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/store.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/store.ts @@ -2,6 +2,7 @@ import { availableParallelism } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { writeJsonAtomic } from "./artifacts.js"; +import type { DeepScanExecutionSettings } from "./recovery-settings.js"; import { boundedDeepScanErrorMessage, DeepScanNonRetryableError, @@ -28,10 +29,12 @@ import type { type JsonObject = Record; export type WorkbenchRunner = ( args: string[], - input?: string + input?: string, + selectFinalization?: boolean, + withExecutionSettings?: boolean, ) => Promise; -const WORKFLOW_VERSION = "deep-scan-mcp/v1"; +const WORKFLOW_VERSION = "deep-security-scan/v2"; const MAX_IDEMPOTENT_PERSISTENCE_ATTEMPTS = 3; const PERSISTENCE_RETRY_BASE_DELAY_MS = 100; @@ -105,6 +108,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { private readonly coordinatorLeases = new Map; }>(); constructor(private readonly runWorkbench: WorkbenchRunner) {} @@ -119,6 +123,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { reasoningEffort?: string; threadId: string; scanRoot: string; + executionSettings?: DeepScanExecutionSettings | null; }): Promise { const userContext = input.userContext; const result = await this.enqueueWrite([ @@ -128,7 +133,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { ...(input.scanId ? ["--scan-id", input.scanId] : []), ...(input.targetPath ? ["--target-path", input.targetPath] : []), ...(input.scope ? ["--scope", input.scope] : []), - ...(userContext ? ["--user-context-stdin"] : []), + ...(userContext && input.executionSettings === undefined ? ["--user-context-stdin"] : []), ...(input.handoffClaimToken ? ["--claim-token", input.handoffClaimToken] : []), ...(input.model ? ["--model", input.model] : []), ...(input.reasoningEffort ? ["--reasoning-effort", input.reasoningEffort] : []), @@ -138,7 +143,10 @@ export class WorkbenchDeepScanStore implements DeepScanStore { String(availableParallelism()), "--workflow-version", WORKFLOW_VERSION - ], false, userContext); + ], false, input.executionSettings === undefined ? userContext : JSON.stringify({ + executionSettings: input.executionSettings, + userContext + }), false, input.executionSettings !== undefined); const run = parseDeepScan(result); const startDisposition = result.startDisposition; if (startDisposition !== "created" && startDisposition !== "joined") { @@ -157,7 +165,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { ])); const generation = this.coordinatorLeases.get(scanId)?.run.coordinatorGeneration; if ( - run.status !== "running" + (run.status !== "running" && !(run.status === "succeeded" && run.finalizationInput)) || (generation !== undefined && run.coordinatorGeneration !== undefined && run.coordinatorGeneration > generation) @@ -176,7 +184,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { input.threadId, ...this.coordinatorLeaseArgs(input.scanId), ...(input.handoffClaimToken ? ["--claim-token", input.handoffClaimToken] : []) - ]); + ], false, undefined, false, true); const run = parseDeepScan(result); const disposition = result.coordinatorDisposition; if (disposition !== "claimed" && disposition !== "adopted" && disposition !== "observing") { @@ -204,13 +212,32 @@ export class WorkbenchDeepScanStore implements DeepScanStore { throw new Error("Deep Scan coordinator lease belongs to another continuation."); } const updatedAt = new Date().toISOString(); - await writeJsonAtomic(join( - lease.run.scanDir, + await this.writeCoordinatorHeartbeat(lease, updatedAt); + return { ...lease.run, updatedAt }; + } + + async releaseCoordinator(scanId: string): Promise { + const lease = this.coordinatorLeases.get(scanId); + if (!lease) return; + this.coordinatorLeases.delete(scanId); + await this.writeCoordinatorHeartbeat(lease, new Date().toISOString(), true); + } + + private writeCoordinatorHeartbeat( + lease: { run: DeepScanRunState; heartbeatWrite?: Promise }, + updatedAt: string, + released = false + ): Promise { + // Heartbeats bypass SQLite writes; only writes to this lease file must settle in order. + const { run } = lease; + const operation = (lease.heartbeatWrite ?? Promise.resolve()).then(() => writeJsonAtomic(join( + run.scanDir, "artifacts", "deep_discovery", - `coordinator-heartbeat-${lease.run.coordinatorGeneration}.json` - ), { coordinatorGeneration: lease.run.coordinatorGeneration, updatedAt }); - return { ...lease.run, updatedAt }; + `coordinator-heartbeat-${run.coordinatorGeneration}.json` + ), { coordinatorGeneration: run.coordinatorGeneration, updatedAt, ...(released ? { released: true } : {}) })); + lease.heartbeatWrite = operation.then(() => undefined, () => undefined); + return operation; } async cancel(scanId: string, threadId: string): Promise { @@ -250,8 +277,10 @@ export class WorkbenchDeepScanStore implements DeepScanStore { ? ["--replaceable-failure-kind", update.replaceableFailureKind] : []) ], true); - const worker = parseWorker(result, update.id); const state = objectValue(result.deepScan, "deepScan"); + const worker = parseWorker(state.workerReceipt + ? { deepScan: { ...state, workers: [state.workerReceipt] } } + : result, update.id); return state.consecutiveErrors === undefined ? worker : { @@ -269,8 +298,8 @@ export class WorkbenchDeepScanStore implements DeepScanStore { workerIds: string[]; promptPath: string; artifactDir: string; - }): Promise { - await this.enqueueWrite([ + }): Promise { + return parseDeepScan(await this.enqueueWrite([ "claim-deep-scan-dedup", "--scan-id", input.scanId, @@ -282,7 +311,7 @@ export class WorkbenchDeepScanStore implements DeepScanStore { input.artifactDir, ...this.coordinatorLeaseArgs(input.scanId), ...input.workerIds.flatMap((workerId) => ["--input-worker-id", workerId]) - ], true); + ], true)); } async commitDedup(commit: DedupCommit): Promise { @@ -303,8 +332,32 @@ export class WorkbenchDeepScanStore implements DeepScanStore { ], true)); } + async selectFinalization(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + resultPath?: string; + omittedWorkerIds: string[]; + }): Promise { + return parseDeepScan(await this.enqueueWrite([ + "finish-deep-scan", + "--scan-id", + input.scanId, + ...(input.coordinatorGeneration === undefined + ? this.coordinatorLeaseArgs(input.scanId) + : ["--coordinator-generation", String(input.coordinatorGeneration)]), + "--terminal-reason", + input.reason, + "--manifest-path", + input.manifestPath, + ...input.omittedWorkerIds.flatMap((workerId) => ["--omitted-worker-id", workerId]) + ], true, JSON.stringify({ resultPath: input.resultPath ?? null }), true)); + } + async finish(input: { scanId: string; + coordinatorGeneration?: number; reason: DeepScanTerminalReason; manifestPath: string; stagedManifestPath?: string; @@ -314,7 +367,9 @@ export class WorkbenchDeepScanStore implements DeepScanStore { "finish-deep-scan", "--scan-id", input.scanId, - ...this.coordinatorLeaseArgs(input.scanId), + ...(input.coordinatorGeneration === undefined + ? this.coordinatorLeaseArgs(input.scanId) + : ["--coordinator-generation", String(input.coordinatorGeneration)]), "--terminal-reason", input.reason, "--manifest-path", @@ -407,13 +462,15 @@ export class WorkbenchDeepScanStore implements DeepScanStore { private enqueueWrite( args: string[], retryTransientFailure = false, - input?: string + input?: string, + selectFinalization = false, + withExecutionSettings = false, ): Promise { const operation = this.writeTail.then(async () => { try { return retryTransientFailure - ? await this.runIdempotentPersistence(args) - : await this.runWorkbench(args, input); + ? await this.runIdempotentPersistence(args, input, selectFinalization) + : await this.runWorkbench(args, input, selectFinalization, withExecutionSettings); } catch (error) { const scanId = argumentValue(args, "--scan-id"); if (scanId && isStaleCoordinatorGenerationError(error)) { @@ -430,11 +487,11 @@ export class WorkbenchDeepScanStore implements DeepScanStore { } /** Replay only existing, same-identity workbench mutations after transient failures. */ - private async runIdempotentPersistence(args: string[]): Promise { + private async runIdempotentPersistence(args: string[], input?: string, selectFinalization = false): Promise { const startedAt = Date.now(); for (let attempt = 1; attempt <= MAX_IDEMPOTENT_PERSISTENCE_ATTEMPTS; attempt += 1) { try { - return await this.runWorkbench(args); + return await this.runWorkbench(args, input, selectFinalization); } catch (error) { if (!isTransientPersistenceError(error)) { throw error; @@ -586,6 +643,10 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { }; return { scanId: requiredString(value.scanId, "deepScan.scanId"), + schemaVersion: optionalPositiveInteger(value.schemaVersion), + workflowVersion: optionalString(value.workflowVersion), + finalizationInput: parseFinalizationInput(value.finalizationInput), + usageOwner: parseUsageOwner(value.usageOwner), status, phase: deepScanPhase(value.phase), coordinatorGeneration: optionalPositiveInteger(value.coordinatorGeneration), @@ -594,6 +655,8 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { targetPath: requiredString(value.targetPath, "deepScan.targetPath"), scope: requiredString(value.scope, "deepScan.scope"), userContext: optionalString(value.userContext), + model: optionalString(value.model), + reasoningEffort: optionalString(value.reasoningEffort), scanDir: requiredString(value.scanDir, "deepScan.scanDir"), config, dispatchedCount: nonNegativeInteger(value.dispatchedCount, "deepScan.dispatchedCount"), @@ -609,7 +672,49 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { : undefined, error: optionalString(value.error), persistedWorkers: parsePersistedWorkers(value.workers), - persistedDedupInputs: parsePersistedDedupInputs(value.dedupInputs) + persistedDedupInputs: parsePersistedDedupInputs(value.dedupInputs), + persistedMergeClaims: Array.isArray(value.mergeClaims) ? value.mergeClaims.map((candidate) => { + const claim = objectValue(candidate, "deepScan.mergeClaim"); + return { + workerId: requiredString(claim.workerId, "deepScan.mergeClaim.workerId"), + previousWorkerId: optionalString(claim.previousWorkerId), + previousResultPath: optionalString(claim.previousResultPath), + previousResultSha256: optionalString(claim.previousResultSha256) + }; + }) : [], + ...(value.committedMerge ? { committedMerge: parseCommittedMerge(value.committedMerge) } : {}) + }; +} + +function parseUsageOwner(value: unknown): DeepScanRunState["usageOwner"] { + if (value === undefined || value === null) return null; + const owner = objectValue(value, "deepScan.usageOwner"); + return { + threadId: optionalString(owner.threadId) ?? null, + turnId: optionalString(owner.turnId) ?? null, + startedAt: requiredString(owner.startedAt, "deepScan.usageOwner.startedAt"), + ...(typeof owner.dedicated === "boolean" ? { dedicated: owner.dedicated } : {}) + }; +} + +function parseFinalizationInput(value: unknown): DeepScanRunState["finalizationInput"] { + if (value === undefined || value === null) return undefined; + const input = objectValue(value, "deepScan.finalizationInput"); + if (input.terminalReason !== "saturated" && input.terminalReason !== "capped") { + throw new Error("Codex Security workbench returned invalid finalization terminal reason."); + } + if (!Array.isArray(input.omittedWorkerIds)) { + throw new Error("Codex Security workbench returned invalid finalization omissions."); + } + return { + version: positiveInteger(input.version, "deepScan.finalizationInput.version"), + resultPath: input.resultPath === null + ? null : requiredString(input.resultPath, "deepScan.finalizationInput.resultPath"), + resultSha256: input.resultSha256 === null + ? null : requiredString(input.resultSha256, "deepScan.finalizationInput.resultSha256"), + terminalReason: input.terminalReason, + omittedWorkerIds: input.omittedWorkerIds.map((id) => requiredString(id, "omittedWorkerId")), + selectedAt: requiredString(input.selectedAt, "deepScan.finalizationInput.selectedAt") }; } @@ -632,11 +737,24 @@ function parsePersistedDedupInputs(value: unknown): PersistedDeepScanDedupInput[ inputOrder: nonNegativeInteger( input.inputOrder, "deepScan.dedupInput.inputOrder" - ) + ), + resultManifestPath: optionalString(input.resultManifestPath), + resultManifestSha256: optionalString(input.resultManifestSha256), + attempt: optionalPositiveInteger(input.attempt) }; }); } +function parseCommittedMerge(value: unknown): NonNullable { + const commit = objectValue(value, "deepScan.committedMerge"); + return { + workerId: requiredString(commit.workerId, "deepScan.committedMerge.workerId"), + resultManifestPath: requiredString(commit.resultManifestPath, "deepScan.committedMerge.resultManifestPath"), + resultManifestSha256: requiredString(commit.resultManifestSha256, "deepScan.committedMerge.resultManifestSha256"), + newFindings: nonNegativeInteger(commit.newFindings, "deepScan.committedMerge.newFindings") + }; +} + function deepScanPhase(value: unknown): DeepScanRunState["phase"] { if (value === undefined || value === null) return undefined; if (value === "setup" || value === "discovery" || value === "reducing" || value === "terminal") { @@ -698,6 +816,7 @@ function parsePersistedWorker(value: JsonObject): PersistedDeepScanWorker { attempt: nonNegativeInteger(value.attempt, "deepScan.worker.attempt"), threadId: optionalString(value.sdkThreadId), resultManifestPath: optionalString(value.resultManifestPath), + acceptedResultPath: optionalString(value.acceptedResultPath), completionSequence: optionalPositiveInteger(value.completionSequence), error: optionalString(value.error) }; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/types.ts b/plugins/codex-security/mcp-app/src/deep-scan/types.ts index dea2d9431..505b3722d 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/types.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/types.ts @@ -1,4 +1,5 @@ import type { DeepReducerContext } from "../artifact-io.js"; +import type { ScanExecutionAttribution } from "../../../../../sdk/typescript/src/scan-sessions.js"; export type DeepScanTerminalReason = "saturated" | "capped"; @@ -37,8 +38,21 @@ export interface DeepScanCanonicalArtifacts { export type DeepScanReducerArtifacts = DeepScanCanonicalArtifacts; +export interface DeepScanFinalizationInput { + version: number; + resultPath: string | null; + resultSha256: string | null; + terminalReason: DeepScanTerminalReason; + omittedWorkerIds: string[]; + selectedAt: string; +} + export interface DeepScanRunState { scanId: string; + schemaVersion?: number; + workflowVersion?: string; + finalizationInput?: DeepScanFinalizationInput; + usageOwner?: ScanExecutionAttribution["owner"] | null; status: DeepScanRunStatus; phase?: "setup" | "discovery" | "reducing" | "terminal"; coordinatorGeneration?: number; @@ -47,6 +61,8 @@ export interface DeepScanRunState { targetPath: string; scope: string; userContext?: string; + model?: string; + reasoningEffort?: string; scanDir: string; config: DeepScanConfig; dispatchedCount: number; @@ -58,12 +74,29 @@ export interface DeepScanRunState { error?: string; persistedWorkers?: PersistedDeepScanWorker[]; persistedDedupInputs?: PersistedDeepScanDedupInput[]; + persistedMergeClaims?: PersistedDeepScanMergeClaim[]; + committedMerge?: { + workerId: string; + resultManifestPath: string; + resultManifestSha256: string; + newFindings: number; + }; +} + +export interface PersistedDeepScanMergeClaim { + workerId: string; + previousWorkerId?: string; + previousResultPath?: string; + previousResultSha256?: string; } export interface PersistedDeepScanDedupInput { dedupWorkerId: string; discoveryWorkerId: string; inputOrder: number; + resultManifestPath?: string; + resultManifestSha256?: string; + attempt?: number; } export interface BeginDeepScanResult { @@ -112,6 +145,7 @@ export interface PersistedDeepScanWorker { attempt: number; threadId?: string; resultManifestPath?: string; + acceptedResultPath?: string; completionSequence?: number; consecutiveErrors?: number; mergeState: DeepScanMergeState; @@ -151,10 +185,19 @@ export interface DeepScanStore { workerIds: string[]; promptPath: string; artifactDir: string; - }): Promise; + }): Promise; commitDedup(commit: DedupCommit): Promise; + selectFinalization?(input: { + scanId: string; + coordinatorGeneration?: number; + reason: DeepScanTerminalReason; + manifestPath: string; + resultPath?: string; + omittedWorkerIds: string[]; + }): Promise; finish(input: { scanId: string; + coordinatorGeneration?: number; reason: DeepScanTerminalReason; manifestPath: string; stagedManifestPath?: string; diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts index 82f119979..107b8f778 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts @@ -1,9 +1,10 @@ -import { createHash } from "node:crypto"; +import type { ScanDraftInput } from "../artifact-scan-draft.js"; +import { auditEvidence, runAcceptedAudit } from "../../../../../sdk/typescript/src/accepted-audit.js"; import { promises as fs } from "node:fs"; import { dirname, join } from "node:path"; -import { getCodexSecurityDeepReducerInputs } from "../artifact-deep-reducer.js"; +import { readDeepReductionSources } from "../artifact-deep-reducer.js"; import { - validateDiscoveryArtifacts, + readDiscoveryAuditDraft, validateReducerArtifacts } from "./artifact-validation.js"; import type { DeepReductionInput, ReducerArtifactValidation } from "./artifact-validation.js"; @@ -41,8 +42,6 @@ export interface AcceptedDiscovery { completionSequence: number; attempt: number; threadId?: string; - basePromptSha256: string; - attemptPromptPaths: string[]; } export type DiscoveryOutcome = @@ -66,8 +65,6 @@ export interface SuccessfulDedupOutcome { newFindings: number; attempt: number; threadId?: string; - basePromptSha256: string; - attemptPromptPaths: string[]; run: DeepScanRunState; } @@ -81,27 +78,12 @@ export interface FailedDedupOutcome { export type DedupOutcome = SuccessfulDedupOutcome | FailedDedupOutcome; -/** Audit evidence for every logical SDK execution, including failures and cancellation. */ -export interface WorkerExecutionAudit { - id: string; - label: string; - kind: DeepScanWorkerKind; - status: "succeeded" | "failed" | "canceled"; - attempt: number; - threadId?: string; - promptPath: string; - artifactDir: string; - basePromptSha256: string; - attemptPromptPaths: string[]; - error?: string; - failureKind?: DeepScanReplaceableFailureKind; -} - export interface ReducerRequest { id: string; label: string; consumed: AcceptedDiscovery[]; previousReducerResultPath?: string; + previousSourceCoverage?: DeepReductionInput["sourceCoverage"]; } export interface DeepScanWorkerRunnerOptions { @@ -115,13 +97,11 @@ export interface DeepScanWorkerRunnerOptions { log: DeepScanLogger; retryDelaysMs: readonly number[]; signal: AbortSignal; - recordExecution?: (execution: WorkerExecutionAudit) => void; } interface WorkerAttemptEvidence { attempt: number; threadId?: string; - attemptPromptPaths: string[]; } type WorkerAttemptOutcome = @@ -185,8 +165,9 @@ export class DeepScanWorkerRunner { artifactContext: { root: artifactDir, layout: "worker" }, subagents: run.config.subagents, validate: async () => { - await validateDiscoveryArtifacts(artifacts, files.resultPath, run.scanId); - discoveryValidated = true; + const draft = await readDiscoveryAuditDraft(artifacts, files.resultPath, run.scanId); + discoveryValidated = draft.complete !== false; + return draft; }, beforeRetry: async (attempt) => { await archiveDirectory( @@ -207,15 +188,6 @@ export class DeepScanWorkerRunner { if (!discoveryValidated) { await fs.rm(files.resultPath, { force: true }); } - const basePromptSha256 = sha256(basePrompt); - this.recordExecution({ - id: workerId, - label: workerLabel, - kind: "discovery", - promptPath, - artifactDir, - basePromptSha256 - }, outcome); if (outcome.status === "failed") { return { type: "discovery", @@ -234,16 +206,6 @@ export class DeepScanWorkerRunner { return { type: "discovery", status: "canceled", workerId }; } - if (this.options.signal.aborted) { - await this.persistWorkerCancellation({ - workerId, - kind: "discovery", - promptPath, - artifactDir - }, outcome.attempt, outcome.threadId); - return { type: "discovery", status: "canceled", workerId }; - } - const acceptance = { id: workerId, scanId: run.scanId, @@ -257,11 +219,7 @@ export class DeepScanWorkerRunner { }; let persisted: PersistedDeepScanWorker; try { - persisted = await this.replayStoreMutation( - "discovery_acceptance_replay", - workerId, - async () => await this.options.store.updateWorker(acceptance) - ); + persisted = await this.options.store.updateWorker(acceptance); } catch (error) { if (!this.options.signal.aborted) throw error; return { type: "discovery", status: "canceled", workerId }; @@ -280,12 +238,10 @@ export class DeepScanWorkerRunner { id: workerId, label: workerLabel, artifactDir, - resultPath: files.resultPath, + resultPath: persisted.acceptedResultPath ?? files.resultPath, completionSequence: persisted.completionSequence, attempt: outcome.attempt, - threadId: outcome.threadId, - basePromptSha256, - attemptPromptPaths: outcome.attemptPromptPaths + threadId: outcome.threadId } }; } @@ -294,9 +250,9 @@ export class DeepScanWorkerRunner { const { id: reducerId, label: reducerLabel, - consumed, - previousReducerResultPath + previousSourceCoverage } = request; + let { consumed, previousReducerResultPath } = request; const { artifacts, run } = this.options; const reducerRoot = join(artifacts.dedupRoot, reducerLabel); const artifactDir = join(reducerRoot, "output"); @@ -312,13 +268,23 @@ export class DeepScanWorkerRunner { })) }); await writePrivateFile(promptPath, basePrompt); - await this.options.store.claimDedup({ + const claimed = await this.options.store.claimDedup({ id: reducerId, scanId: run.scanId, workerIds: consumed.map((worker) => worker.id), promptPath, artifactDir }); + const claim = claimed?.persistedMergeClaims?.find((item) => item.workerId === reducerId); + if (claim) { + previousReducerResultPath = claim.previousResultPath; + const inputs = claimed?.persistedDedupInputs?.filter((item) => item.dedupWorkerId === reducerId) ?? []; + consumed = inputs.sort((a, b) => a.inputOrder - b.inputOrder).map((item) => { + const discovery = consumed.find((worker) => worker.id === item.discoveryWorkerId); + if (!discovery || !item.resultManifestPath) throw new Error("The reducer claim is missing an accepted input."); + return { ...discovery, resultPath: item.resultManifestPath, attempt: item.attempt ?? discovery.attempt }; + }); + } this.options.log({ event: "dedup_claimed", scanId: run.scanId, @@ -326,6 +292,7 @@ export class DeepScanWorkerRunner { count: consumed.length }); + const persistSourceCoverage = "workflowVersion" in run && run.workflowVersion === "deep-security-scan/v2"; const artifactContext = { root: artifactDir, repoRoot: run.targetPath, @@ -333,13 +300,17 @@ export class DeepScanWorkerRunner { layout: "reducer" as const, deepReducer: { scanRoot: artifacts.scanDir, - claimedWorkers: consumed.map((worker) => ({ id: worker.id, resultPath: worker.resultPath })), + persistSourceCoverage, + claimedWorkers: consumed.map((worker) => ({ id: worker.id, resultPath: worker.resultPath, artifactDir: worker.artifactDir, attempt: worker.attempt })), previousReducerResultPath } }; // Snapshot inputs before execution: direct file output has the same // conservation checks as the MCP writer without rereading consumed sources. - const sources = await getCodexSecurityDeepReducerInputs(artifactContext); + const sources = await readDeepReductionSources(artifactContext); + if (sources.previous && previousSourceCoverage !== undefined) { + sources.previous.sourceCoverage = structuredClone(previousSourceCoverage); + } let reducerValidation: ReducerArtifactValidation | undefined; let outcome = await this.runWorkerWithRetries({ workerId: reducerId, @@ -356,7 +327,8 @@ export class DeepScanWorkerRunner { resultPath, reducerId, previousReducerResultPath, - sources + sources, + persistSourceCoverage }, run.scanId); }, beforeRetry: async (attempt) => { @@ -377,15 +349,6 @@ export class DeepScanWorkerRunner { }, outcome.attempt, outcome.threadId); outcome = { ...outcome, status: "canceled" }; } - const basePromptSha256 = sha256(basePrompt); - this.recordExecution({ - id: reducerId, - label: reducerLabel, - kind: "dedup", - promptPath, - artifactDir, - basePromptSha256 - }, outcome); if (outcome.status === "failed") { if (outcome.error instanceof DeepScanNonRetryableError) throw outcome.error; return { @@ -397,58 +360,37 @@ export class DeepScanWorkerRunner { }; } if (outcome.status === "canceled") throw abortError(); - if (this.options.signal.aborted) { - await this.persistWorkerCancellation({ - workerId: reducerId, - kind: "dedup", - promptPath, - artifactDir - }, outcome.attempt, outcome.threadId); - throw abortError(this.options.signal.reason); - } - if (!reducerValidation) { throw new Error(`${reducerId} completed without validated reducer artifacts.`); } - if (this.options.signal.aborted) { - await this.persistWorkerCancellation({ - workerId: reducerId, - kind: "dedup", - promptPath, - artifactDir - }, outcome.attempt, outcome.threadId); - throw abortError(this.options.signal.reason); - } - const commit = { id: reducerId, scanId: run.scanId, newFindings: reducerValidation.newFindings, resultManifestPath: resultPath }; - const committed = await this.replayStoreMutation( - "dedup_commit_replay", - reducerId, - async () => await this.options.store.commitDedup(commit) - ); + const committed = await this.options.store.commitDedup(commit); + const accepted = committed.committedMerge; + const acceptedPath = accepted?.resultManifestPath ?? resultPath; + // V1 checkpoints omit host-only coverage; retain the validated projection. + const acceptedResult = reducerValidation.result; + const newFindings = accepted?.newFindings ?? reducerValidation.newFindings; this.options.log({ event: "dedup_committed", scanId: run.scanId, workerId: reducerId, count: consumed.length, - newFindings: reducerValidation.newFindings + newFindings }); return { type: "dedup", id: reducerId, consumed, - resultPath, - result: reducerValidation.result, - newFindings: reducerValidation.newFindings, + resultPath: acceptedPath, + result: acceptedResult, + newFindings, attempt: outcome.attempt, threadId: outcome.threadId, - basePromptSha256, - attemptPromptPaths: outcome.attemptPromptPaths, run: committed }; } @@ -461,7 +403,7 @@ export class DeepScanWorkerRunner { artifactDir: string; artifactContext?: CodexWorkerArtifactContext; subagents: number; - validate: () => Promise; + validate: () => Promise; beforeRetry: (attempt: number) => Promise; }): Promise { const { run, signal } = this.options; @@ -470,10 +412,9 @@ export class DeepScanWorkerRunner { let continuationPrompt: string | undefined; let lastThreadId: string | undefined; let executionPromptPath = input.promptPath; - const attemptPromptPaths = [input.promptPath]; for (let attempt = 1; attempt <= maximumAttempts; attempt += 1) { if (signal.aborted) { - return await this.cancelAttempt(input, attempt, lastThreadId, attemptPromptPaths); + return await this.cancelAttempt(input, attempt, lastThreadId); } let validationStarted = false; let validationCompleted = false; @@ -497,7 +438,7 @@ export class DeepScanWorkerRunner { attempt }); try { - const result = await this.options.executor.run({ + const execute = () => this.options.executor.run({ kind: input.kind, promptPath: executionPromptPath, // Discovery workers write only to their isolated directory. Setup and @@ -526,19 +467,36 @@ export class DeepScanWorkerRunner { }); } }); - if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); - } - validationStarted = true; - try { - await input.validate(); - } catch (validationError) { - throw withWorkerDiagnostics(validationError, result.diagnostics); - } - validationCompleted = true; - if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); + const accept = async (result: Awaited>) => { + validationStarted = true; + let accepted: ScanDraftInput | void; + try { + accepted = await input.validate(); + } catch (validationError) { + throw withWorkerDiagnostics(validationError, result.diagnostics); + } + validationCompleted = accepted?.complete !== false; + return accepted === undefined ? {} : auditEvidence(accepted); + }; + // Reducers keep their aggregate contract; discovery uses the shared audit. + const audit = input.kind === "discovery" + ? await runAcceptedAudit({ signal, execute, accept }) + : undefined; + let result: Awaited>; + if (audit) { + if (audit.status === "checkpoint") { + throw withWorkerDiagnostics( + new Error("Standard scan worker wrote only a checkpoint; its audit is not complete."), + audit.execution.diagnostics, + ); + } + result = audit.execution; + } else { + result = await execute(); + if (signal.aborted) return await this.cancelAttempt(input, attempt, activeThreadId); + await accept(result); } + if (signal.aborted) return await this.cancelAttempt(input, attempt, activeThreadId); this.options.log({ event: "worker_succeeded", scanId: run.scanId, @@ -549,12 +507,11 @@ export class DeepScanWorkerRunner { return { status: "succeeded", attempt, - threadId: result.threadId ?? activeThreadId, - attemptPromptPaths: [...attemptPromptPaths] + threadId: result.threadId ?? activeThreadId }; } catch (error) { if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); + return await this.cancelAttempt(input, attempt, activeThreadId); } const normalized = asError(error); const policyRefusal = input.kind === "discovery" @@ -588,8 +545,7 @@ export class DeepScanWorkerRunner { ? {} : { consecutiveErrors: persistedFailure.consecutiveErrors }), attempt, - threadId: activeThreadId, - attemptPromptPaths: [...attemptPromptPaths] + threadId: activeThreadId }; } await this.options.store.updateWorker({ @@ -627,7 +583,6 @@ export class DeepScanWorkerRunner { failedAttempt: attempt, error: normalized }); - attemptPromptPaths.push(executionPromptPath); } } const delayMs = Math.ceil( @@ -645,7 +600,7 @@ export class DeepScanWorkerRunner { await this.options.clock.sleep(delayMs, signal); } catch (sleepError) { if (signal.aborted) { - return await this.cancelAttempt(input, attempt, activeThreadId, attemptPromptPaths); + return await this.cancelAttempt(input, attempt, activeThreadId); } throw sleepError; } @@ -680,32 +635,6 @@ export class DeepScanWorkerRunner { }); } - /** Replay idempotent SQLite commits when their process response is ambiguous. */ - private async replayStoreMutation( - event: string, - workerId: string, - operation: () => Promise - ): Promise { - try { - return await operation(); - } catch (firstError) { - this.options.log({ - event, - scanId: this.options.run.scanId, - workerId, - reason: errorKind(firstError) - }); - try { - return await operation(); - } catch (replayError) { - throw new Error( - `Deep Scan persistence replay failed: ${asError(replayError).message}`, - { cause: firstError } - ); - } - } - } - private async cancelAttempt( input: { workerId: string; @@ -714,36 +643,15 @@ export class DeepScanWorkerRunner { artifactDir: string; }, attempt: number, - threadId: string | undefined, - attemptPromptPaths: string[] + threadId: string | undefined ): Promise { await this.persistWorkerCancellation(input, attempt, threadId); return { status: "canceled", attempt, - threadId, - attemptPromptPaths: [...attemptPromptPaths] + threadId }; } - - private recordExecution( - input: Omit, - outcome: WorkerAttemptOutcome - ): void { - this.options.recordExecution?.({ - ...input, - status: outcome.status, - attempt: outcome.attempt, - ...(outcome.threadId ? { threadId: outcome.threadId } : {}), - attemptPromptPaths: [...outcome.attemptPromptPaths], - ...(outcome.status === "failed" ? { - error: outcome.error.message, - ...(outcome.replaceableFailureKind - ? { failureKind: outcome.replaceableFailureKind } - : {}) - } : {}) - }); - } } /** @@ -880,22 +788,10 @@ function validationErrorData(error: Error, message: string): Record { @@ -297,7 +297,7 @@ export function registerCompactWorkerArtifactTools( registerCompactTool(server, { name: "record_codex_security_scan_draft", title: "Record Codex Security Scan Draft", - description: "Save this Standard worker's semantic findings and coverage. Use complete:false for progress checkpoints, then complete:true for its final result; keep unvalidated candidates in coverage.deferred.", + description: "Save this Standard worker's semantic findings and coverage. Use complete:false for progress checkpoints, then complete:true for its final result; keep unvalidated candidates in coverage.deferred. Provide the required top-level scanId. The workbench supplies the following fields; omit them from your draft: findingId, occurrenceId, and fingerprints within each finding; includePaths and excludePaths within scope; and documentType, schemaVersion, scanId, mode, includePaths, excludePaths, receiptRefs, and inventoryStrategy within coverage.", inputSchema: scanDraftInputSchema, readOnly: false, handler: async (value) => recordCodexSecurityWorkerScanDraft( diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs new file mode 100644 index 000000000..a0af00fa9 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs @@ -0,0 +1,315 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { build } from "esbuild"; + +const pluginRoot = fileURLToPath(new URL("../../", import.meta.url)); +const exec = promisify(execFile); +const bundled = await build({ + bundle: true, + stdin: { + contents: [ + 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";', + 'export { createDeepScanArtifacts } from "./src/deep-scan/artifacts.ts";', + 'export { validateDiscoveryArtifacts } from "./src/deep-scan/artifact-validation.ts";', + 'export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts";', + 'export { createScanArtifactContext } from "./src/artifact-context.ts";', + 'export { recordCodexSecurityScanDraftViaWorkbench, saveScanDraftCheckpoint } from "./src/artifact-scan-draft.ts";', + 'export { recordCodexSecurityDeepReduction, getCodexSecurityDeepReducerInputs } from "./src/artifact-deep-reducer.ts";', + ].join("\n"), + resolveDir: path.join(pluginRoot, "mcp-app"), + }, + format: "esm", platform: "node", loader: { ".md": "text" }, write: false, +}); +export async function publishCoverageFixture(root, completeness, { + resume = false, + continueAfterResume = false, + immutableInputs = false, + materialFindings = false, + discardMutableResults = false, + legacyAttempts = false, + splitSeededReducers = false, + selectedRecovery = false, + sourceProvenance, +} = {}) { + const runtimePath = path.join(root, "fixture-runtime.mjs"); + await writeFile(runtimePath, bundled.outputFiles[0].contents); + const { DeepScanCoordinator, createDeepScanArtifacts, validateDiscoveryArtifacts, WorkbenchDeepScanStore, createScanArtifactContext, recordCodexSecurityScanDraftViaWorkbench, recordCodexSecurityDeepReduction, getCodexSecurityDeepReducerInputs, saveScanDraftCheckpoint } = await import(pathToFileURL(runtimePath).href); + const targetPath = path.join(root, "target"); + const codexHome = path.join(root, "codex-home"); + const scanRoot = path.join(root, "scans"); + const threadId = "coverage-fixture-owner"; + const statuses = completeness === "partial" ? ["partial", "complete", "unknown"] + : completeness === "unknown" ? ["unknown", "complete"] : ["complete"]; + await mkdir(scanRoot, { mode: 0o700 }); + await mkdir(targetPath, { recursive: true }); + await mkdir(path.join(codexHome, "codex-security"), { recursive: true }); + await writeFile(path.join(targetPath, "source.py"), "# Synthetic source\n"); + await writeFile(path.join(codexHome, "codex-security", "config.toml"), + `[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = ${statuses.length}\nmax_discovery_runs = ${statuses.length}\n`); + const runWorkbench = async (args, input, selectFinalization = false) => { + const script = path.join(pluginRoot, "scripts", "workbench_db.py"); + const pythonArgs = selectFinalization + ? ["-c", "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", script, ...args] + : [script, ...args]; + const execution = exec(process.env.PYTHON || "python3", pythonArgs, { + env: { ...process.env, CODEX_HOME: codexHome, CODEX_SECURITY_STATE_DIR: path.join(root, "state") }, + }); + if (input !== undefined) execution.child.stdin.end(input); + const { stdout } = await execution; + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + let { run } = await store.begin({ targetPath, scope: ".", threadId, scanRoot }); + assert.equal(run.workflowVersion, "deep-security-scan/v2", "new scans use persisted finalization"); + if (selectedRecovery) { + ({ run } = await store.claimCoordinator({ scanId: run.scanId, threadId })); + } else { + // Seed an existing v1 run for legacy direct publication and in-memory coverage recovery. + await exec(process.env.PYTHON || "python3", ["-c", [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + " db.execute(\"UPDATE deep_scan_runs SET workflow_version = 'deep-scan-mcp/v1' WHERE scan_id = ?\", (sys.argv[2],))", + ].join("\n"), path.join(root, "state", "workbench.sqlite3"), run.scanId]); + run = await store.get(run.scanId, threadId); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + } + const context = await createScanArtifactContext(run.scanId, runWorkbench, { requireRunning: true }); + const rawSources = new Map(); + const writeReduction = async (context) => { + const inputs = await getCodexSecurityDeepReducerInputs(context); + const sources = [...(inputs.previous?.findings ?? []), ...inputs.discoveries.flatMap((source) => source.result.findings)]; + const findings = []; + if (sources.length) { + const finding = structuredClone(sources[0]); + finding.provenance.sourceFindingIds = [...new Set(sources.flatMap((source) => source.provenance.sourceFindingIds))]; + delete finding.provenance.sourceFindings; + findings.push(finding); + } + await recordCodexSecurityDeepReduction(context, { scanId: run.scanId, findings }); + }; + const writeDiscovery = async (artifactDir, index) => { + const status = statuses[index]; + const pending = completeness === "partial" && status !== "complete"; + const coverage = { + completeness: status, + surfaces: [{ id: "shared-surface", label: "Archive route", disposition: pending ? "needs_follow_up" : "no_issue_found", receiptRefs: ["artifacts/review.md"] }], + explicitExclusions: [{ pattern: "vendor/", reason: "External dependency." }], + deferred: pending ? [{ id: "same-id", candidateId: "candidate-1", reason: index === 0 ? "Verify entry boundaries." : "Verify symbolic links.", paths: ["source.py"], surfaceIds: ["shared-surface"] }] : [], + openQuestions: pending ? [{ question: `Deployment question ${index + 1}.` }] : [], + }; + if (sourceProvenance !== undefined) { + for (const field of ["surfaces", "explicitExclusions", "deferred", "openQuestions"]) { + for (const item of coverage[field]) item.provenance = structuredClone(sourceProvenance); + } + } + await mkdir(path.join(artifactDir, "artifacts"), { recursive: true }); + await writeFile(path.join(artifactDir, "artifacts", "review.md"), "Synthetic review evidence.\n"); + const resultPath = path.join(artifactDir, "result.json"); + const findings = materialFindings && index < 2 ? [{ + ruleId: "archive-extraction", identity: { anchor: "archive-destination" }, + title: "Archive entries can escape the destination", + summary: "Archive extraction requires both entry containment and symbolic-link handling.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Synthetic accepted source evidence." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "source.py", startLine: 1, endLine: 1 }], + remediation: materialRemediations[index], + remediationTests: [materialRemediationTests[index]], + provenance: { source: "local_plugin" }, + }] : []; + const bytes = JSON.stringify({ scanId: run.scanId, complete: true, findings, coverage }); + await writeFile(resultPath, bytes); + rawSources.set(resultPath, bytes); + if (immutableInputs) { + await saveScanDraftCheckpoint({ root: artifactDir, repoRoot: targetPath, layout: "worker" }, JSON.parse(bytes)); + const head = JSON.parse(await readFile(path.join(artifactDir, "checkpoint-head.json"), "utf8")); + const acceptedPath = path.join(artifactDir, "checkpoints", head.checkpoint); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + return acceptedPath; + } + return resultPath; + }; + if (resume) { + const workers = []; + const seeded = continueAfterResume ? statuses.slice(0, -1) : statuses; + for (const index of seeded.keys()) { + const workerRoot = path.join(run.scanDir, "artifacts", "deep_discovery", "workers", `discovery-${String(index + 1).padStart(4, "0")}`); + const artifactDir = path.join(workerRoot, "output"); + const worker = { id: randomUUID(), scanId: run.scanId, kind: "discovery", promptPath: path.join(workerRoot, "prompt.md"), artifactDir, attempt: index === 0 ? 2 : 1 }; + const writtenPath = await writeDiscovery(artifactDir, index); + const resultManifestPath = discardMutableResults ? path.join(artifactDir, "result.json") : writtenPath; + await writeFile(worker.promptPath, "Synthetic discovery prompt.\n"); + for (const status of ["queued", "running", "succeeded"]) { + await store.updateWorker({ ...worker, status, ...(status === "succeeded" ? { resultManifestPath } : {}) }); + } + workers.push({ ...worker, resultPath: resultManifestPath }); + } + const batches = splitSeededReducers ? [workers.slice(0, 2), workers.slice(2)] : [workers]; + let lastReducerId; + let lastReducerReference; + for (const [index, batch] of batches.entries()) { + const label = `dedup-${String(index + 1).padStart(4, "0")}`; + const artifactDir = path.join(run.scanDir, "artifacts", "deep_discovery", "dedup", label, "output"); + const promptPath = path.join(path.dirname(artifactDir), "prompt.md"); + await mkdir(artifactDir, { recursive: true }); + await writeFile(promptPath, "Synthetic reducer prompt.\n"); + const id = randomUUID(); + const claimed = await store.claimDedup({ id, scanId: run.scanId, workerIds: batch.map((worker) => worker.id), artifactDir, promptPath }); + if (discardMutableResults) { + await store.updateWorker({ id, scanId: run.scanId, kind: "dedup", status: "running", artifactDir, promptPath, attempt: 1 }); + } + const resultManifestPath = path.join(artifactDir, "result.json"); + // Legacy accepted reducers omitted coverage entirely. + if (materialFindings) { + await writeReduction({ + root: artifactDir, repoRoot: targetPath, scanId: run.scanId, layout: "reducer", + deepReducer: { + scanRoot: run.scanDir, + claimedWorkers: batch.map((worker) => { + const input = claimed.persistedDedupInputs.find((input) => input.dedupWorkerId === id && input.discoveryWorkerId === worker.id); + return { ...worker, resultPath: input.resultManifestPath ?? worker.resultPath, attempt: input.attempt ?? worker.attempt }; + }), + persistSourceCoverage: selectedRecovery, + previousReducerResultPath: claimed.persistedMergeClaims?.find((claim) => claim.workerId === id)?.previousResultPath, + }, + }); + } else { + await writeFile(resultManifestPath, JSON.stringify({ scanId: run.scanId, findings: [] })); + } + rawSources.set(resultManifestPath, await readFile(resultManifestPath, "utf8")); + const committed = await store.commitDedup({ id, scanId: run.scanId, newFindings: materialFindings && index === 0 ? 1 : 0, resultManifestPath }); + lastReducerReference = committed.committedMerge.resultManifestPath; + lastReducerId = id; + } + if (legacyAttempts) { + // Migrated discoveries and prior reducers can have frozen claims without attempt rows. + await exec(process.env.PYTHON || "python3", ["-c", [ + "import sqlite3, sys", + "with sqlite3.connect(sys.argv[1]) as db:", + " db.execute(\"DELETE FROM deep_scan_attempts WHERE worker_id != ?\", (sys.argv[2],))", + ].join("\n"), path.join(root, "state", "workbench.sqlite3"), lastReducerId]); + } + run = await store.get(run.scanId, threadId); + if (discardMutableResults) { + for (const worker of run.persistedWorkers) { + const acceptedPath = worker.acceptedResultPath ?? run.persistedDedupInputs + .find((input) => input.discoveryWorkerId === worker.id)?.resultManifestPath + ?? run.persistedMergeClaims.find((claim) => claim.previousWorkerId === worker.id)?.previousResultPath; + assert.ok(acceptedPath, "the real store retains an accepted reference"); + assert.notEqual(acceptedPath, worker.resultManifestPath); + rawSources.set(acceptedPath, await readFile(acceptedPath, "utf8")); + rawSources.delete(worker.resultManifestPath); + await rm(worker.resultManifestPath); + } + } + if (selectedRecovery) { + run = await store.selectFinalization({ + scanId: run.scanId, reason: "capped", manifestPath: path.join(run.scanDir, "scan-manifest.json"), + resultPath: lastReducerReference, omittedWorkerIds: [], + }); + } + } + let discoveryCalls = 0; + const executor = { + async run(request) { + assert.equal(resume && !continueAfterResume, false, "accepted legacy sources should resume without new model work"); + const thread = request.resumeThreadId ?? randomUUID(); + await request.onThreadStarted?.(thread); + if (request.kind === "discovery") { + discoveryCalls++; + const index = Number(path.basename(path.dirname(request.promptPath)).split("-").at(-1)) - 1; + if (index === 0 && !request.resumeThreadId) return { threadId: thread, finalResponse: "Continue the unfinished audit." }; + await writeDiscovery(request.artifactContext.root, index); + } else { + if (immutableInputs) { + const current = await store.get(run.scanId, threadId); + for (const claimed of request.artifactContext.deepReducer.claimedWorkers) { + const accepted = current.persistedWorkers.find((worker) => worker.id === claimed.id); + assert.equal(claimed.resultPath, accepted.acceptedResultPath ?? accepted.resultManifestPath, "the reducer uses the exact accepted input"); + assert.equal(claimed.artifactDir, accepted.artifactDir, "receipts retain their original output owner"); + } + } + await writeReduction({ ...request.artifactContext, repoRoot: targetPath, scanId: run.scanId }); + } + return { threadId: thread, finalResponse: "Audit finished." }; + }, + }; + let publicationCalls = 0; + const options = { + run, store, executor, pluginRoot, retryDelaysMs: [1], + onComplete: async (draft, signal, publication) => { + publicationCalls++; + if (selectedRecovery && publicationCalls === 1) throw new Error("Synthetic selected publication failure"); + await recordCodexSecurityScanDraftViaWorkbench(context, draft, runWorkbench, signal, selectedRecovery ? publication : undefined); + }, + }; + const coordinator = new DeepScanCoordinator(options); + coordinator.start(); + let terminal; + if (selectedRecovery) { + await assert.rejects(coordinator.wait(undefined, 30_000), /Synthetic selected publication failure/); + const pending = await store.get(run.scanId, threadId); + assert.equal(pending.status, "running"); + assert.deepEqual(pending.finalizationInput, run.finalizationInput); + const worker = pending.persistedWorkers.find((worker) => worker.kind === "discovery"); + const rejected = { scanId: run.scanId, complete: false, findings: [], coverage: { + completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], + } }; + await saveScanDraftCheckpoint({ root: worker.artifactDir, repoRoot: targetPath, layout: "worker" }, rejected); + const replacement = path.join(worker.artifactDir, "result.json"); + await writeFile(replacement, JSON.stringify(rejected)); + await assert.rejects(validateDiscoveryArtifacts(createDeepScanArtifacts(run.scanDir), replacement, run.scanId), /only a checkpoint/); + const headPath = path.join(worker.artifactDir, "checkpoint-head.json"); + const head = JSON.parse(await readFile(headPath, "utf8")); + for (const file of [replacement, headPath, path.join(worker.artifactDir, "checkpoints", head.checkpoint)]) { + rawSources.set(file, await readFile(file, "utf8")); + } + const restarted = new DeepScanCoordinator({ ...options, run: pending }); + restarted.start(); + terminal = await restarted.wait(undefined, 30_000); + assert.deepEqual(terminal.finalizationInput, run.finalizationInput); + assert.equal(publicationCalls, 2); + } else { + terminal = await coordinator.wait(undefined, 30_000); + } + assert.equal(terminal?.status, "succeeded", terminal?.error); + assert.equal(terminal.noNewStreak, materialFindings ? (resume && !continueAfterResume && !splitSeededReducers ? 0 : 1) : statuses.length, + "source coverage must not change stopping policy"); + assert.equal(discoveryCalls, resume ? (continueAfterResume ? 1 : 0) : statuses.length + 1); + const accepted = await store.get(run.scanId, threadId); + for (const worker of accepted.persistedWorkers.filter((worker) => worker.kind === "dedup")) { + const resultPath = worker.acceptedResultPath + ?? accepted.persistedMergeClaims.find((claim) => claim.previousWorkerId === worker.id)?.previousResultPath + ?? worker.resultManifestPath; + const result = JSON.parse(await readFile(resultPath, "utf8")); + assert.equal(Object.hasOwn(result, "sourceCoverage"), selectedRecovery, "coverage persistence follows the accepted workflow version"); + if (!rawSources.has(worker.resultManifestPath)) { + for (const name of await readdir(path.join(worker.artifactDir, "checkpoints"))) { + const checkpoint = JSON.parse(await readFile(path.join(worker.artifactDir, "checkpoints", name), "utf8")); + assert.equal(Object.hasOwn(checkpoint, "sourceCoverage"), selectedRecovery, "checkpoint coverage follows the accepted workflow version"); + } + } + } + await runWorkbench(["complete-scan", "--scan-id", run.scanId]); + for (const [file, bytes] of rawSources) assert.equal(await readFile(file, "utf8"), bytes); + return { scanDir: run.scanDir, threadId, terminal }; +} + +export const materialRemediations = [ + "Check the destination before writing the archive entry.", + "Reject symbolic links before opening the destination.", +]; +export const materialRemediationTests = [ + "Reject an archive entry outside the destination.", + "Reject a symbolic link inside the destination.", +]; + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const result = await publishCoverageFixture(process.argv[2], process.argv[3], { resume: process.argv[4] === "true", continueAfterResume: process.argv[5] === "true" }); + process.stdout.write(JSON.stringify(result)); +} diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs b/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs index 5e44d1796..ea36d9f67 100644 --- a/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs +++ b/plugins/codex-security/mcp-app/tests/deep_scan_publication_cases.mjs @@ -1,11 +1,91 @@ import assert from "node:assert/strict"; -import { readFile, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; export async function testDeepScanPublication({ fixtureRun, FakeStore, FakeExecutor, DeepScanCoordinator, deferred, immediateClock, eventually, }) { + async function testSelectedFinalizationOwner() { + for (const failure of [undefined, "draft", "finish", "parent", "replacement"]) { + const fixture = await fixtureRun({ workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 1 }); + fixture.run.workflowVersion = "deep-security-scan/v2"; + fixture.run.coordinatorGeneration = 3; + fixture.run.finalizationInput = { + version: 1, resultPath: null, resultSha256: null, terminalReason: "capped", + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + }; + const store = new FakeStore(fixture.run); + const finishStarted = deferred(); + const releaseFinish = deferred(); + const parentStarted = deferred(); + const releaseParent = deferred(); + const calls = []; + const events = []; + store.finish = async () => { + calls.push("finish"); + finishStarted.resolve(); + await releaseFinish.promise; + if (failure === "finish") throw new Error("injected selected finish failure"); + if (failure === "replacement") { + store.run = { ...store.run, status: "succeeded", coordinatorGeneration: 4 }; + throw new Error("Deep Scan coordinator generation is stale."); + } + store.run = { ...store.run, status: "succeeded", terminalReason: "capped" }; + return structuredClone(store.run); + }; + const coordinator = new DeepScanCoordinator({ + run: fixture.run, store, pluginRoot: fixture.pluginRoot, + executor: { run: async () => assert.fail("Selected finalization cannot start model work") }, + threadId: "selected-owner", log: (event) => events.push(event), + onComplete: async () => { + calls.push("draft"); + if (failure === "draft") throw new Error("injected selected draft failure"); + }, + onFinalized: async (run) => { + calls.push("parent"); + assert.equal(run.status, "succeeded"); + assert.equal(run.coordinatorGeneration, 3); + parentStarted.resolve(); + await releaseParent.promise; + if (failure === "parent") throw new Error("injected selected parent failure"); + }, + }); + // Rejections stay observed even when the user detaches their only waiter. + const settled = coordinator.settled(); + settled.catch(() => {}); + const observer = new AbortController(); + coordinator.start(); + const detached = coordinator.wait(observer.signal); + observer.abort(); + await assert.rejects(detached, { name: "AbortError" }); + if (failure !== "draft") { + await finishStarted.promise; + assert.deepEqual(calls, ["draft", "finish"]); + assert.equal(await coordinator.wait(undefined, 10), undefined); + releaseFinish.resolve(); + } + if (failure === undefined || failure === "parent") { + await parentStarted.promise; + assert.equal(await coordinator.wait(undefined, 10), undefined, "wait includes enclosing completion"); + releaseParent.resolve(); + } + if (failure && failure !== "replacement") { + await assert.rejects(settled, new RegExp(`injected selected ${failure} failure`)); + assert.equal(store.failCalls, 0, "publication errors retain the original selected stop"); + assert.equal(events.some((event) => event.event === "coordinator_publication_pending"), true); + assert.equal(store.run.status, failure === "parent" ? "succeeded" : "running"); + } else { + assert.equal((await settled).status, "succeeded"); + } + assert.deepEqual(calls, failure === "draft" ? ["draft"] + : failure === "finish" || failure === "replacement" ? ["draft", "finish"] + : ["draft", "finish", "parent"]); + assert.deepEqual(store.run.finalizationInput, fixture.run.finalizationInput); + } + } + async function testSaturationOmitsWorkerAcceptedDuringCancellation() { const fixture = await fixtureRun({ workers: 3, subagents: 0, stopAfterNoNew: 2, maxDiscoveryRuns: 3 }); const store = new FakeStore(fixture.run); @@ -55,7 +135,7 @@ export async function testDeepScanPublication({ assert.deepEqual(completed[0].findings, [], "late worker findings are not appended to the saturated aggregate"); } - async function testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus() { + async function testSuccessfulDeepCoveragePreservesWorkerReviewStatus() { const fixture = await fixtureRun({ workers: 2, subagents: 0, stopAfterNoNew: 2, maxDiscoveryRuns: 2 }); const store = new FakeStore(fixture.run); const executor = new FakeExecutor(); @@ -87,9 +167,16 @@ export async function testDeepScanPublication({ const terminal = await coordinator.wait(undefined, 5_000); assert.equal(terminal?.status, "succeeded", terminal?.error); assert.equal(completed.length, 1); - assert.deepEqual(completed[0].coverage, { - completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], - }); + assert.equal(completed[0].coverage.completeness, "partial"); + assert.equal(completed[0].coverage.deferred.length, 2); + assert.equal(completed[0].coverage.surfaces.length, 4); + assert.equal(completed[0].coverage.reviews.length, 2); + assert.deepEqual(new Set(completed[0].coverage.reviews.map((review) => review.completeness)), + new Set(["partial", "unknown"])); + for (const item of completed[0].coverage.deferred) { + assert.equal(item.provenance.attempt, 1); + assert.ok(store.workers.has(item.provenance.workerId)); + } for (const worker of store.workers.values()) { if (worker.kind !== "discovery") continue; const draft = JSON.parse(await readFile(worker.resultManifestPath, "utf8")); @@ -101,7 +188,21 @@ export async function testDeepScanPublication({ async function testSaturationIgnoresDiscoveryCancellationWriteFailure() { const fixture = await fixtureRun({ workers: 2, subagents: 0, stopAfterNoNew: 2, maxDiscoveryRuns: 6 }); + fixture.run.workflowVersion = "deep-security-scan/v2"; const store = new FakeStore(fixture.run); + store.selectFinalization = async (input) => { + const checkpointRoot = path.join(path.dirname(input.resultPath), "checkpoints"); + const [name] = await readdir(checkpointRoot); + const checkpoint = path.join(checkpointRoot, name); + const bytes = await readFile(checkpoint); + store.run.finalizationInput = { + version: 1, resultPath: path.relative(fixture.run.scanDir, checkpoint), + resultSha256: createHash("sha256").update(bytes).digest("hex"), + terminalReason: input.reason, omittedWorkerIds: input.omittedWorkerIds, + selectedAt: "2026-01-01T00:00:00Z", + }; + return structuredClone(store.run); + }; const executor = new FakeExecutor({ blockDedup: true, blockDiscoveryAfterCalls: 2 }); const updateWorker = store.updateWorker.bind(store); const rejectedCancellations = new Set(); @@ -142,7 +243,7 @@ export async function testDeepScanPublication({ )); const { coverage, ...publishedReduction } = completed[0]; assert.deepEqual( - publishedReduction, + { ...publishedReduction, sourceCoverage: coverage }, JSON.parse(await readFile(acceptedReducer.resultManifestPath, "utf8")), "the accepted aggregate still reaches publication when redundant cancellation writes fail", ); @@ -150,6 +251,7 @@ export async function testDeepScanPublication({ async function testPublicationUsesAcceptedReducerSnapshot() { const fixture = await fixtureRun({ workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 1 }); + fixture.run.coordinatorGeneration = 3; const store = new FakeStore(fixture.run); const commitDedup = store.commitDedup.bind(store); store.commitDedup = async (commit) => { @@ -163,21 +265,31 @@ export async function testDeepScanPublication({ return structuredClone(store.run); }; const completed = []; + const published = []; const coordinator = new DeepScanCoordinator({ run: fixture.run, store, executor: new FakeExecutor({ discoveryCandidateId: "accepted-finding" }), pluginRoot: fixture.pluginRoot, clock: immediateClock, - onComplete: async (draft) => completed.push(structuredClone(draft)), + onComplete: async (draft, _signal, publication) => { + completed.push(structuredClone(draft)); + published.push(publication); + }, }); coordinator.start(); const terminal = await coordinator.wait(undefined, 5_000); assert.equal(terminal?.status, "succeeded", terminal?.error); assert.equal(completed[0].findings[0].provenance.candidateId, "accepted-finding"); assert.equal(completed[0].coverage.completeness, "complete"); + const reducer = [...store.workers.values()].find((worker) => worker.kind === "dedup"); + assert.deepEqual(published, [{ + coordinatorGeneration: 3, + resultPath: reducer.resultManifestPath, + }]); } + await testSelectedFinalizationOwner(); await testSaturationOmitsWorkerAcceptedDuringCancellation(); - await testSuccessfulDeepCoverageIgnoresWorkerAndReducerReviewStatus(); + await testSuccessfulDeepCoveragePreservesWorkerReviewStatus(); await testSaturationIgnoresDiscoveryCancellationWriteFailure(); await testPublicationUsesAcceptedReducerSnapshot(); } diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs new file mode 100644 index 000000000..0fed15b23 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_bank.mjs @@ -0,0 +1,180 @@ +// Synthetic accepted artifacts. Expected fixes are independent of display identities. +export const scanId = "7fc17317-9594-49e0-b06a-d72fd7e14bba"; +export const bankVersion = "accepted-source-bank/v1"; + +export const fixes = { + owner: "Check document ownership before returning document contents.", + path: "Resolve archive paths and reject entries outside the extraction directory.", + sql: "Bind the search term as a SQL parameter.", + html: "HTML-encode the search term before rendering the response.", +}; + +function finding(id, remediation, extra = {}) { + return { + ruleId: "synthetic." + id, + identity: { anchor: id }, + title: "Request boundary " + id, + summary: "A request-controlled value crosses an unchecked boundary.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "A synthetic local test reaches the sink.", + }, + taxonomy: { category: "input-validation", cwe: ["CWE-20"] }, + locations: [{ path: "src/routes.py", startLine: 12, endLine: 14 }], + remediation, + provenance: { source: "local_plugin" }, + ...extra, + }; +} + +function coverage(completeness = "complete", deferred = []) { + return { completeness, surfaces: [], explicitExclusions: [], deferred }; +} + +function worker(id, findings, scanCoverage = coverage()) { + return { + id, + result: { scanId, complete: true, findings, coverage: scanCoverage }, + }; +} + +export const workers = [ + worker( + "worker-owner", + [ + finding("owner", fixes.owner, { + summary: + "An authenticated user can read another user's document by changing its ID.", + validation: { + summary: "The ownership check is absent; authentication is required.", + }, + }), + ], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check the alternate document handler.", + paths: ["src/alternate.py"], + }, + ]), + ), + worker( + "worker-path", + [finding("path", fixes.path)], + coverage("partial", [ + { + candidateId: "candidate-1", + reason: "Check symlink extraction separately.", + paths: ["src/archive.py"], + }, + ]), + ), + worker("worker-owner-duplicate", [ + finding("owner-copy", fixes.owner, { + summary: "Document contents may be reachable without authentication.", + severity: { level: "critical" }, + confidence: { + level: "low", + rationale: "Authentication middleware was not examined.", + }, + validation: { + summary: + "The unauthenticated claim is untested; ownership check is absent.", + }, + }), + ]), + worker("worker-bundled", [ + finding("search-bundle", fixes.sql + " " + fixes.html, { + summary: + "The search route interpolates the query into SQL and separately into HTML.", + locations: [{ path: "src/search.py", startLine: 5, endLine: 9 }], + }), + ]), +]; + +export const sourceGroups = { + "worker-owner:0": "owner", + "worker-path:0": "path", + "worker-owner-duplicate:0": "owner", + "worker-bundled:0": "search-bundle", +}; + +export const sourceFixes = { + "worker-owner:0": ["owner"], + "worker-path:0": ["path"], + "worker-owner-duplicate:0": ["owner"], + "worker-bundled:0": ["sql", "html"], +}; + +// This history is distinct from the immutable accepted terminal bank above. +// A newer rejection is authoritative for this logical worker's final result. +export const rejectionHistory = { + workerId: "worker-rejected", + earlier: worker("worker-rejected", [finding("safe-query", fixes.sql)]).result, + latest: worker("worker-rejected", [], { + ...coverage(), + surfaces: [ + { + label: "Search SQL", + disposition: "rejected", + notes: + "The driver binds parameters; the earlier interpolation claim was disproved.", + }, + ], + }).result, +}; + +export function permutations(items) { + if (!items.length) return [[]]; + return items.flatMap((item, index) => + permutations(items.filter((_, i) => i !== index)).map((rest) => [ + item, + ...rest, + ]), + ); +} + +export function partitions(items) { + if (!items.length) return [[]]; + return items.flatMap((_, index) => + partitions(items.slice(index + 1)).map((rest) => [ + items.slice(0, index + 1), + ...rest, + ]), + ); +} + +export function originals(inputs) { + const sources = new Map(); + for (const current of inputs.previous?.findings ?? []) { + for (const source of current.provenance.sourceFindings ?? []) + sources.set(source.id, source.finding); + } + for (const discovery of inputs.discoveries) { + discovery.result.findings.forEach((value, i) => + sources.set(`${discovery.workerId}:${i}`, value), + ); + } + return sources; +} + +// Scripted proposals isolate host reconciliation from model variability. +export function proposal(inputs, groupFor = (id) => sourceGroups[id]) { + const grouped = new Map(); + for (const [id, value] of originals(inputs)) { + const group = groupFor(id); + if (!grouped.has(group)) grouped.set(group, { value, refs: [] }); + grouped.get(group).refs.push(id); + } + return { + scanId, + complete: true, + findings: [...grouped].map(([group, { value, refs }]) => ({ + ...structuredClone(value), + ruleId: "synthetic." + group, + identity: { anchor: group }, + provenance: { source: "local_plugin", sourceFindingIds: refs }, + })), + }; +} diff --git a/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs new file mode 100644 index 000000000..26b6a6163 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/fixtures/accepted_source_replay.mjs @@ -0,0 +1,105 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; +import { proposal, scanId, workers } from "./accepted_source_bank.mjs"; + +export async function loadReducer() { + const bundled = await build({ + bundle: true, + entryPoints: [ + fileURLToPath( + new URL("../../src/artifact-deep-reducer.ts", import.meta.url), + ), + ], + format: "esm", + platform: "node", + write: false, + }); + return import( + "data:text/javascript;base64," + + Buffer.from(bundled.outputFiles[0].contents).toString("base64") + ); +} + +export async function prepareReplay(root, bank = workers) { + const workerRoot = path.join(root, "artifacts", "deep_discovery", "workers"); + const records = new Map(); + for (const [i, worker] of bank.entries()) { + const output = path.join(workerRoot, worker.id, "output"); + await mkdir(output, { recursive: true }); + const resultPath = path.join(output, "result.json"); + await writeFile(resultPath, JSON.stringify(worker.result) + "\n"); + records.set(worker.id, { + id: worker.id, + resultPath, + completionSequence: i + 1, + }); + } + return records; +} + +export async function runBatch( + reducer, + root, + records, + ids, + index, + previous, + propose = proposal, +) { + const output = path.join( + root, + "artifacts", + "deep_discovery", + "dedup", + `dedup-${index}`, + "output", + ); + await mkdir(output, { recursive: true }); + const context = { + root: output, + repoRoot: root, + scanId, + layout: "reducer", + deepReducer: { + scanRoot: root, + claimedWorkers: ids.map((id) => records.get(id)), + ...(previous ? { previousReducerResultPath: previous } : {}), + }, + }; + const inputs = await reducer.getCodexSecurityDeepReducerInputs(context); + const submitted = propose(inputs); + const receipt = await reducer.recordCodexSecurityDeepReduction( + context, + submitted, + ); + const resultPath = path.join(output, "result.json"); + return { + inputs, + submitted, + receipt, + resultPath, + result: JSON.parse(await readFile(resultPath, "utf8")), + }; +} + +export async function replay(reducer, root, batches, propose = proposal) { + const records = await prepareReplay(root); + const steps = []; + let previous; + for (const [index, batch] of batches.entries()) { + const step = await runBatch( + reducer, + root, + records, + batch, + index, + previous, + propose, + ); + steps.push(step); + previous = step.resultPath; + } + return { steps, result: steps.at(-1).result }; +} diff --git a/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs new file mode 100644 index 000000000..2b8709773 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_accepted_source_replay.mjs @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + partitions, + permutations, + sourceFixes, + workers, +} from "./fixtures/accepted_source_bank.mjs"; +import { loadReducer, replay } from "./fixtures/accepted_source_replay.mjs"; + +test("accepted sources survive completion orders and eligible batch partitions", async () => { + const reducer = await loadReducer(); + const root = await realpath( + await mkdtemp(path.join(tmpdir(), "accepted-source-replay-")), + ); + let count = 0; + try { + for (const order of permutations(workers.map((worker) => worker.id))) { + for (const batches of partitions(order)) { + // Ordinary rolling runs require two successes for the first merge. + if (batches[0].length < 2) continue; + const runRoot = path.join(root, String(count++)); + const { result, steps } = await replay(reducer, runRoot, batches); + assert.equal(result.findings.length, 3); + const refs = result.findings.flatMap( + (finding) => finding.provenance.sourceFindingIds, + ); + assert.deepEqual(refs.toSorted(), Object.keys(sourceFixes).toSorted()); + for (const step of steps) { + assert.deepEqual( + step.receipt.consumedWorkerIds, + batches[steps.indexOf(step)], + ); + } + for (const finding of result.findings) { + for (const source of finding.provenance.sourceFindings) { + const [workerId, index] = source.id.split(":"); + const original = workers.find((worker) => worker.id === workerId) + .result.findings[Number(index)]; + assert.deepEqual(source.finding, original); + } + } + for (const worker of workers) { + const persisted = JSON.parse( + await readFile( + path.join( + runRoot, + "artifacts", + "deep_discovery", + "workers", + worker.id, + "output", + "result.json", + ), + "utf8", + ), + ); + assert.deepEqual(persisted, worker.result); + } + } + } + assert.equal(count, 96); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs index 5466999cc..c9cb7533d 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_deep_reducer.mjs @@ -14,7 +14,8 @@ const bundled = await build({ const { deepReducerInputsInputSchema, deepReductionInputSchema, - getCodexSecurityDeepReducerInputs, + getCodexSecurityDeepReducerInputs: getModelInputs, + readDeepReductionSources: getCodexSecurityDeepReducerInputs, recordCodexSecurityDeepReduction } = await import( "data:text/javascript;base64," @@ -99,6 +100,7 @@ try { layout: "reducer", deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [first, second] } }; @@ -127,13 +129,23 @@ try { /evidenceRefs must refer/, "live reducer submissions reject unknown evidence references instead of silently removing them", ); - assert.deepEqual(inputs, { + assert.deepEqual({ ...inputs, discoveries: inputs.discoveries.map(({ coverage, ...source }) => source) }, { discoveries: [ { workerId: first.id, result: withSourceRefs(first) }, { workerId: second.id, result: withSourceRefs(second) } ], previous: null }); + assert.equal(inputs.discoveries[0].coverage.completeness, "partial"); + assert.equal(inputs.discoveries[1].coverage.completeness, "unknown"); + assert.deepEqual(await getModelInputs(context), { + discoveries: inputs.discoveries.map(({ workerId, result }) => ({ workerId, result })), + previous: null, + }, "coverage accounting does not change reducer model inputs"); + assert.deepEqual(inputs.discoveries[0].coverage.deferred[0].provenance, + { workerId: first.id, candidateId: "candidate-upload" }); + assert.deepEqual(inputs.discoveries[0].coverage.surfaces[0].receiptRefs, + ["artifacts/deep_discovery/workers/discovery-0001/output/artifacts/missing-worker-receipt.md"]); assert.equal(JSON.stringify(inputs).includes(root), false); assert.equal(JSON.stringify(inputs).includes("result.json"), false); @@ -158,6 +170,10 @@ try { const outcome = await recordCodexSecurityDeepReduction(context, merged); const mergedWithSources = { ...merged, + sourceCoverage: { + ...inputs.discoveries[0].coverage, + reviews: [...inputs.discoveries[0].coverage.reviews, ...inputs.discoveries[1].coverage.reviews], + }, findings: [ retainedFinding(shared, [{ id: "worker-001:0", finding: shared }, { id: "worker-002:0", finding: shared }]), retainedFinding(independent, [{ id: "worker-002:1", finding: independent }]), @@ -176,7 +192,7 @@ try { assert.deepEqual( JSON.parse(await readFile(path.join(outputRoot, "checkpoints", checkpointNames[0]), "utf8")), mergedWithSources, - "reducer checkpoints retain the accepted findings and scope without coverage", + "reducer checkpoints retain accepted findings, scope and source coverage", ); assert.deepEqual( @@ -234,12 +250,16 @@ try { layout: "reducer", deepReducer: { scanRoot, + persistSourceCoverage: true, claimedWorkers: [third], previousReducerResultPath: path.join(outputRoot, "result.json") } }; const nextInputs = await getCodexSecurityDeepReducerInputs(nextContext); - assert.deepEqual(nextInputs, { + const { sourceCoverage, ...previousModelInput } = mergedWithSources; + assert.deepEqual((await getModelInputs(nextContext)).previous, previousModelInput, + "host coverage metadata is excluded from the previous model input too"); + assert.deepEqual({ ...nextInputs, discoveries: nextInputs.discoveries.map(({ coverage, ...source }) => source) }, { discoveries: [{ workerId: third.id, result: withSourceRefs(third) }], previous: mergedWithSources }); @@ -255,6 +275,10 @@ try { JSON.parse(await readFile(path.join(nextOutputRoot, "result.json"), "utf8")), { ...mergedWithSources, + sourceCoverage: { + ...mergedWithSources.sourceCoverage, + reviews: [...mergedWithSources.sourceCoverage.reviews, ...nextInputs.discoveries[0].coverage.reviews], + }, findings: [ retainedFinding(shared, [ { id: "worker-003:0", finding: shared }, diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs index 72f815fc3..aa5859ad2 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.mjs @@ -591,6 +591,10 @@ try { const obsoleteCheckpointPath = path.join(deepParentRoot, "checkpoints", "obsolete.json"); await writeFile(obsoleteCheckpointPath, "{malformed obsolete checkpoint\n"); let deepWorkbenchWrites = 0; + const deepPublication = { + coordinatorGeneration: 3, + resultPath: path.join(deepParentRoot, "workers", "reducer", "result.json"), + }; await recordCodexSecurityScanDraftViaWorkbench( deepParentContext, acceptedDeepDraft, @@ -603,11 +607,15 @@ try { const checkpointPath = arguments_[arguments_.indexOf("--checkpoint-path") + 1]; const staged = JSON.parse(await readFile(draftPath, "utf8")); const stagedCheckpoint = JSON.parse(await readFile(checkpointPath, "utf8")); + assert.deepEqual(staged.deepScanPublication, deepPublication); + assert.equal(stagedCheckpoint.deepScanPublication, undefined); assert.deepEqual(staged.findings, acceptedDeepFindings); assert.deepEqual(staged.coverage, acceptedDeepCoverage); assert.deepEqual(stagedCheckpoint.findings, acceptedDeepDraft.findings); assert.equal(stagedCheckpoint.handoffClaimToken, undefined); }, + undefined, + deepPublication, ); assert.equal(deepWorkbenchWrites, 1, "terminal Deep drafts still publish through the workbench lock despite obsolete malformed checkpoints"); assert.deepEqual(await readdir(path.join(deepParentRoot, "drafts")), []); diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs index 0e5b33efd..3a3009955 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_storage.mjs @@ -21,6 +21,7 @@ try { await writeFile(path.join(repository, "example.py"), "value = 1\n"); await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { __dirname: JSON.stringify(applicationRoot), "import.meta.url": "__filename" }, entryPoints: [path.join(applicationRoot, "main.ts")], external: ["fsevents"], diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs b/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs index 3ce725944..53059a459 100644 --- a/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs +++ b/plugins/codex-security/mcp-app/tests/test_artifact_storage_regressions.mjs @@ -29,6 +29,7 @@ await fs.mkdir(repository); await fs.writeFile(path.join(repository, "example.py"), "value = 1\n"); await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { __dirname: JSON.stringify(applicationRoot), "import.meta.url": "__filename" }, entryPoints: [path.join(applicationRoot, "main.ts")], external: ["fsevents"], format: "cjs", loader: { ".md": "text" }, diff --git a/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs new file mode 100644 index 000000000..8d1e108d1 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_audit_acceptance_contract.mjs @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, readFile, readdir, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/artifact-scan-draft.ts"; + export * from "./src/deep-scan/artifact-validation.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "../../../sdk/typescript/src/accepted-audit.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=audit-acceptance-contract.js" }, +}); +const { + createDeepScanArtifacts, recordCodexSecurityScanDraft, + recordCodexSecurityWorkerScanDraft, validateDiscoveryArtifacts, + readDiscoveryAuditDraft, auditEvidence, runAcceptedAudit, +} = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`); + +const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; +const finding = { + ruleId: "path-traversal.archive-extraction", title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "src/extract.py", startLine: 4 }], + remediation: "Validate the resolved output path before writing.", + provenance: { source: "local_plugin", candidateId: "archive-entry" }, +}; + +for (const completeness of ["complete", "partial", "unknown"]) { + test(`Standard and Deep retain accepted semantic evidence with ${completeness} coverage`, async () => { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "audit-contract-"))); + try { + const repository = path.join(root, "repository"); + const scanDir = path.join(root, "scan"); + const artifacts = createDeepScanArtifacts(scanDir); + const workerRoot = path.join(artifacts.workersRoot, "discovery-0001", "output"); + await Promise.all([mkdir(repository), mkdir(workerRoot, { recursive: true })]); + const semantic = { + scanId, complete: true, + threatModel: { summary: "An untrusted caller supplies archive entries." }, + findings: [finding], + coverage: { + completeness, + surfaces: [{ id: "archive", label: "Archive extraction", disposition: "reported", receiptRefs: [] }], + explicitExclusions: [], + deferred: completeness === "complete" ? [] : [{ id: "deployment-controls", reason: "Deployment controls remain unverified." }], + }, + }; + const standard = { + root: scanDir, repoRoot: repository, layout: "scan", scanId, + mode: "standard", status: "running", scope: ".", + targetContract: { + target: { + allowedKinds: ["directory_snapshot"], targetId: "target_example", displayName: "example", + requiredSnapshotDigest: `codex-security-snapshot/v1:sha256:${"a".repeat(64)}`, + }, + scope: { requiredIncludePaths: ["."], requiredExcludePaths: [] }, diffTarget: null, + }, + }; + const worker = { root: workerRoot, repoRoot: repository, layout: "worker", scanId }; + const checkpoint = { ...semantic, complete: false }; + await recordCodexSecurityScanDraft(standard, checkpoint); + await recordCodexSecurityWorkerScanDraft(worker, checkpoint); + await assert.rejects(validateDiscoveryArtifacts(artifacts, path.join(workerRoot, "result.json"), scanId), /checkpoint/); + assert.equal(JSON.parse(await readFile(path.join(scanDir, "scan-manifest.json"))).scan.complete, false); + const controller = new AbortController(); + const execute = async () => ({ threadId: "audit-conversation", usage: null }); + const accept = async () => auditEvidence(await readDiscoveryAuditDraft( + artifacts, path.join(workerRoot, "result.json"), scanId, + )); + const unfinished = await runAcceptedAudit({ signal: controller.signal, execute, accept }); + assert.equal(unfinished.status, "checkpoint"); + assert.equal(unfinished.checkpoint.complete, false); + assert.equal(unfinished.accepted, undefined); + assert.equal(unfinished.execution.usage, null); + const standardWrite = await recordCodexSecurityScanDraft(standard, semantic); + const deepWrite = await recordCodexSecurityWorkerScanDraft(worker, semantic); + assert.equal(standardWrite.status, "draft_written"); + assert.equal(deepWrite.status, "draft_written"); + const accepted = await validateDiscoveryArtifacts(artifacts, path.join(workerRoot, "result.json"), scanId); + const manifest = JSON.parse(await readFile(path.join(scanDir, "scan-manifest.json"))); + const findings = JSON.parse(await readFile(path.join(scanDir, "findings.json"))); + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"))); + assert.deepEqual(accepted.findings, [finding]); + for (const [key, value] of Object.entries(finding)) { + assert.deepEqual(findings.findings[0][key], value); + } + assert.ok(findings.findings[0].identity.anchor); + assert.deepEqual(accepted.threatModel, manifest.scan.threatModel); + for (const field of ["completeness", "surfaces", "explicitExclusions", "deferred"]) { + assert.deepEqual(coverage[field], accepted.coverage[field]); + } + assert.equal(accepted.scanId, scanId); + const audit = await runAcceptedAudit({ signal: controller.signal, execute, accept }); + assert.equal(audit.status, "accepted"); + assert.deepEqual(audit.accepted, accepted); + assert.deepEqual(audit.checkpoint, accepted); + const failure = new Error("Synthetic execution failure"); + await assert.rejects(runAcceptedAudit({ signal: controller.signal, + execute: async () => { throw failure; }, + accept: async () => { assert.fail("An execution failure cannot accept old output"); }, + }), (error) => error === failure); + await assert.rejects(runAcceptedAudit({ signal: controller.signal, execute, + accept: async () => { const evidence = await accept(); controller.abort("user canceled"); return evidence; }, + }), (error) => error === controller.signal.reason); + assert.equal(manifest.scan.sealedAt, undefined); + assert.equal(manifest.scan.artifacts, undefined); + assert.equal((await readdir(workerRoot)).includes("scan-manifest.json"), false); + assert.equal((await readdir(scanDir)).includes("report.md"), false); + await assert.rejects(validateDiscoveryArtifacts(artifacts, path.join(workerRoot, "result.json"), "b4c84677-5aaf-410c-88d2-3e97e6f8c4d8"), /scan/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs index aa7937bd5..285080822 100644 --- a/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs +++ b/plugins/codex-security/mcp-app/tests/test_compact_artifact_server.mjs @@ -203,10 +203,8 @@ async function testCompactDiffScanCompletion(bundle, runtimeLabel) { }), `${runtimeLabel}: record compact diff canonical semantics` ); - requireSuccessfulTool( - await call("complete_codex_security_scan", { scanId, handoffClaimToken }), - `${runtimeLabel}: complete compact diff scan` - ); + const completion = await call("complete_codex_security_scan", { scanId, handoffClaimToken }); + requireSuccessfulTool(completion, `${runtimeLabel}: complete compact diff scan`); const completed = requireSuccessfulTool( await call("get_codex_security_completed_scan", { scanId, handoffClaimToken }), `${runtimeLabel}: read completed compact diff scan` @@ -219,6 +217,9 @@ async function testCompactDiffScanCompletion(bundle, runtimeLabel) { ); assert.equal(completed.coverage.inventoryStrategy, "diff"); assert.equal(completed.findings.findings.length, 0); + await assertCompletionCoverageAndReplay( + call, { scanId, handoffClaimToken }, completion, completed.coverage + ); } finally { await client.close(); } @@ -591,10 +592,13 @@ async function testSemanticScanDraftCompletion(bundle, runtimeLabel) { assert.equal(reporting.status, "running"); assert.deepEqual(reporting.phaseProgress, { completed: 0, total: 0, unit: null }); - const completed = requireSuccessfulTool(await call( + const completion = await call( "complete_codex_security_scan", { scanId, handoffClaimToken } - ), `${runtimeLabel}: finalize the accepted draft exactly once`); + ); + const completed = requireSuccessfulTool( + completion, `${runtimeLabel}: finalize the accepted draft exactly once` + ); assert.equal(completed.scan.progress.status, "complete"); assert.equal(completed.scan.reportAvailable, true); @@ -623,6 +627,9 @@ async function testSemanticScanDraftCompletion(bundle, runtimeLabel) { assert.equal(results.manifest.scan.target.kind, "directory_snapshot"); assert.equal(results.coverage.inventoryStrategy, "directory"); assert.equal(results.coverage.completeness, "partial"); + await assertCompletionCoverageAndReplay( + call, { scanId, handoffClaimToken }, completion, results.coverage + ); assert.deepEqual(results.coverage.includePaths, ["."]); assert.deepEqual(results.coverage.excludePaths, []); assert.equal(results.coverage.surfaces[0].disposition, "reported"); @@ -676,6 +683,33 @@ async function testSemanticScanDraftCompletion(bundle, runtimeLabel) { } } +async function assertCompletionCoverageAndReplay(call, identity, completion, coverage) { + const expected = { + completeness: coverage.completeness, + surfaceCount: coverage.surfaces.length, + deferredCount: coverage.deferred.length, + explicitExclusionCount: coverage.explicitExclusions.length + }; + const scan = requireSuccessfulTool(completion).scan; + const snapshot = () => Promise.all([ + "scan-manifest.json", "findings.json", "coverage.json", "report.md" + ].map((name) => readFile(path.join(scan.scanDir, name), "utf8"))); + const sealed = await snapshot(); + const replay = await call("complete_codex_security_scan", identity); + for (const result of [completion, replay]) { + const structured = requireSuccessfulTool(result); + assert.equal(structured.scan.progress.status, "complete"); + assert.equal(result.content[0].type, "text"); + assert.match(result.content[0].text, new RegExp(`Canonical coverage: ${expected.completeness}\\b`)); + assert.match(result.content[0].text, new RegExp(`\\b${expected.surfaceCount} surfaces\\b`)); + assert.match(result.content[0].text, new RegExp(`\\b${expected.deferredCount} deferred items\\b`)); + assert.match(result.content[0].text, new RegExp(`\\b${expected.explicitExclusionCount} explicit exclusions\\b`)); + assert.deepEqual(structured.coverageSummary, expected); + assert.deepEqual(structured.scan.usage, scan.usage); + } + assert.deepEqual(await snapshot(), sealed, "completion replay preserves sealed artifacts"); +} + async function snapshotScanDraft(scanDirectory) { return Promise.all(["scan-manifest.json", "findings.json", "coverage.json"].map( async (artifact) => { @@ -1109,6 +1143,18 @@ async function testDiscoveryWorkerToolList(bundle) { } }; + for (const arguments_ of [ + { ...input, scope: { includePaths: ["src"], excludePaths: [] } }, + { ...input, coverage: { ...input.coverage, scanId } } + ]) { + requireToolError( + await client.callTool({ name: tool.name, arguments: arguments_ }), + /expected never/, + "The draft description must not change rejection of workbench-owned metadata." + ); + await assert.rejects(readFile(resultPath), { code: "ENOENT" }); + } + requireToolError( await client.callTool({ name: tool.name, @@ -1229,6 +1275,7 @@ async function testReducerWorkerToolList(bundle) { async function bundleEntrypoint(entrypoint, outfile) { await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { __dirname: JSON.stringify(applicationRoot), "import.meta.url": "__filename" diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs index 2b0bd4391..79c0e7bd8 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.mjs @@ -20,6 +20,7 @@ const bundle = await build({ write: false }); const { + readDiscoveryAuditDraft, validateDiscoveryArtifacts, validateReducerArtifacts } = await import( @@ -34,12 +35,63 @@ try { await testDiscoveryValidation(root); await testReducerValidation(root); await testEmptyDiscoveryAndReduction(root); + await testWriteupReferences(root); } finally { await rm(root, { recursive: true, force: true }); } console.log("deep scan artifact validation tests passed"); +async function testWriteupReferences(root) { + const artifacts = await createLayout(path.join(root, "writeups")); + const reportPath = "findings/example/example.md"; + const result = draft([{ + ...finding("example", "src/a.js"), + writeup: { reportPath }, + }]); + const worker = await createWorker(artifacts, "discovery-0001", "worker-001", result); + const artifactDir = path.join(artifacts.dedupRoot, "dedup-0001", "output"); + const resultPath = path.join(artifactDir, "result.json"); + await mkdir(artifactDir, { recursive: true }); + const { coverage: _coverage, ...reduction } = result; + await writeResult(resultPath, reduction); + const validateDiscovery = () => validateDiscoveryArtifacts(artifacts, worker.resultPath, scanId); + const validateReducer = () => validateReducerArtifacts({ + artifacts, artifactDir, resultPath, reducerId: "dedup-0001", + sources: { discoveries: [{ workerId: worker.id, result: reduction }], previous: null }, + }, scanId); + + for (const validate of [validateDiscovery, validateReducer]) { + await assert.rejects(validate(), /findings\[0\]\.writeup\.reportPath/); + } + assert.deepEqual(JSON.parse(await readFile(resultPath, "utf8")), reduction, + "rejected results remain available as original attempt evidence"); + + await writeResult(worker.resultPath, { ...result, complete: false }); + assert.equal((await readDiscoveryAuditDraft(artifacts, worker.resultPath, scanId)).complete, false); + await writeResult(worker.resultPath, result); + + const workerReport = path.join(path.dirname(worker.resultPath), reportPath); + await mkdir(path.dirname(workerReport), { recursive: true }); + await writeFile(workerReport, "# Worker-local report\n"); + await assert.rejects(validateDiscovery(), /writeup\.reportPath/, + "publication resolves report paths from the parent scan directory"); + + const parentReport = path.join(artifacts.scanDir, reportPath); + await mkdir(path.dirname(parentReport), { recursive: true }); + await writeFile(parentReport, ""); + assert.deepEqual((await validateDiscovery()).findings[0].writeup, { reportPath }); + assert.deepEqual((await validateReducer()).result.findings[0].writeup, { reportPath }, + "valid references are retained, including empty regular files accepted by publication"); + + if (process.platform !== "win32") { + await rm(parentReport); + await symlink(workerReport, parentReport); + await assert.rejects(validateDiscovery(), /writeup\.reportPath/); + await assert.rejects(validateReducer(), /writeup\.reportPath/); + } +} + async function testDiscoveryValidation(root) { const artifacts = await createLayout(path.join(root, "discovery")); const result = draft([finding("shared", "src/a.js")], { @@ -217,8 +269,9 @@ async function testReducerValidation(root) { discoveries: [{ workerId: first.id, result: draft([firstFinding, secondFinding]) }], previous: null, }; - const validateSnapshot = () => validateReducerArtifacts({ + const validateSnapshot = (persistSourceCoverage = false) => validateReducerArtifacts({ artifacts, artifactDir, resultPath, reducerId: "dedup-0001", sources, + persistSourceCoverage, }, scanId); await assert.rejects(validateSnapshot(), /unaccounted source findings/); await writeResult(resultPath, draft([firstFinding, secondFinding])); @@ -226,11 +279,15 @@ async function testReducerValidation(root) { const validatedSnapshot = await validateSnapshot(); assert.equal(validatedSnapshot.newFindings, 2); const admitted = JSON.parse(await readFile(resultPath, "utf8")); + const { sourceCoverage, ...legacySnapshot } = validatedSnapshot.result; + assert.equal(sourceCoverage.completeness, "unknown"); assert.deepEqual( - validatedSnapshot.result, + legacySnapshot, admitted, - "validation returns the same reconciled result that was accepted on disk", + "v1 preserves host coverage in memory while retaining the legacy persisted shape", ); + const versionedSnapshot = await validateSnapshot(true); + assert.deepEqual(versionedSnapshot.result, JSON.parse(await readFile(resultPath, "utf8")), "v2 persists the full host projection"); assert.equal(Object.hasOwn(admitted, "coverage"), false); assert.deepEqual(admitted.findings[1].provenance.sourceFindingIds, ["worker-001:1"]); sources.previous = structuredClone(admitted); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs new file mode 100644 index 000000000..9a45820a6 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_attempt_replay.mjs @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; + +const app = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const plugin = path.resolve(app, ".."); +const bundle = await build({ + bundle: true, format: "esm", platform: "node", write: false, + loader: { ".md": "text" }, + stdin: { resolveDir: app, contents: [ + 'export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts";', + 'export { DeepScanWorkerRunner } from "./src/deep-scan/worker-runner.ts";', + 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";', + 'export { createDeepScanArtifacts, ensureDeepScanDirectories } from "./src/deep-scan/artifacts.ts";' + ].join("\n") } +}); +const { WorkbenchDeepScanStore, DeepScanWorkerRunner, DeepScanCoordinator, createDeepScanArtifacts, ensureDeepScanDirectories } = + await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`); +const execute = promisify(execFile); +for (const responseLosses of [3, 1]) await testResponseLoss(responseLosses); + +async function testResponseLoss(responseLosses) { + const root = await mkdtemp(path.join(tmpdir(), "deep-attempt-receipt-")); + const target = path.join(root, "target"); + const environment = { ...process.env, CODEX_HOME: path.join(root, "home"), CODEX_SECURITY_STATE_DIR: path.join(root, "state") }; + const counts = new Map(); + const receipts = new Map(); + const raw = async (args) => { + const { stdout } = await execute(process.env.PYTHON || "python3", [path.join(plugin, "scripts/workbench_db.py"), ...args], { + env: environment, timeout: 30_000, maxBuffer: 4 * 1024 * 1024 + }); + return JSON.parse(stdout); + }; + const store = new WorkbenchDeepScanStore(async (args) => { + const result = await raw(args); + const key = args[0] === "commit-deep-scan-dedup" ? "merge" + : args[0] === "upsert-deep-scan-worker" && args[args.indexOf("--status") + 1] === "succeeded" ? "acceptance" : null; + const operation = key === "acceptance" ? `${key}:${args[args.indexOf("--worker-id") + 1]}` : key; + if (key) { + counts.set(key, (counts.get(key) ?? 0) + 1); + const receipt = key === "acceptance" ? result.deepScan.workerReceipt : result.deepScan.committedMerge; + if (!receipts.has(operation)) receipts.set(operation, receipt); + else assert.deepEqual(receipt, receipts.get(operation), "replay returns the original operation receipt"); + if (counts.get(key) <= responseLosses) { + const error = new Error("fixture response lost after committed write"); + error.code = "ETIMEDOUT"; + throw error; + } + } + return result; + }); + try { + await mkdir(target); + await writeFile(path.join(target, "fixture.py"), "print('fixture')\n"); + const { run } = await store.begin({ targetPath: target, scope: ".", threadId: "fixture-owner", scanRoot: path.join(root, "scans") }); + const artifacts = createDeepScanArtifacts(run.scanDir); + await ensureDeepScanDirectories(artifacts); + let executions = 0; + const runner = new DeepScanWorkerRunner({ + run, store, artifacts, pluginRoot: plugin, signal: new AbortController().signal, + random: () => 0.5, log: () => {}, retryDelaysMs: [], + clock: { now: () => Date.now(), sleep: async () => {} }, + executor: { async run(request) { + executions++; + if (request.kind === "dedup") { + const workers = request.artifactContext.deepReducer.claimedWorkers; + const prompt = await readFile(request.promptPath, "utf8"); + const configuration = JSON.parse(prompt.match(/```json\n([\s\S]*?)\n```/)[1]); + assert.deepEqual(configuration.claimedWorkerIds, workers.map(worker => worker.id)); + assert.equal(workers.every(worker => worker.resultPath.includes("checkpoints")), true); + assert.deepEqual(workers.map(worker => worker.attempt), [1, 1], "execution uses the immutable claim attempts"); + } + await request.onThreadStarted?.(`fixture-session-${executions}`); + const draft = { scanId: run.scanId, findings: [], threatModel: { summary: "Synthetic fixture." } }; + if (request.kind === "discovery") draft.coverage = { + completeness: "complete", surfaces: [{ label: "Fixture", disposition: "no_issue_found" }], explicitExclusions: [], deferred: [] + }; + await writeFile(path.join(request.artifactContext.root, "result.json"), JSON.stringify(draft)); + return { threadId: `fixture-session-${executions}` }; + } } + }); + if (responseLosses === 3) { + const outcome = await runner.runDiscoveryWorker(randomUUID(), "discovery-1").catch((error) => error); + assert.equal(counts.get("acceptance"), 3, "the runner must not multiply the store's retry policy"); + assert.match(outcome.message, /response lost/); + assert.equal(executions, 1); + return; + } + const discovery = await runner.runDiscoveryWorker(randomUUID(), "discovery-1"); + assert.equal(discovery.status, "succeeded"); + assert.match(discovery.worker.resultPath, /checkpoints/); + const second = await runner.runDiscoveryWorker(randomUUID(), "discovery-2"); + // Acceptance receipts are operation-specific, so only the first discovery loses a response. + await rm(path.join(discovery.worker.artifactDir, "result.json")); + const merged = await runner.runReducer({ + id: randomUUID(), label: "dedup-1", + consumed: [discovery.worker, second.worker].map(worker => ({ + ...worker, resultPath: path.join(worker.artifactDir, "result.json"), attempt: 99 + })) + }); + assert.equal(merged.error, undefined, merged.error?.stack); + assert.match(merged.resultPath, /checkpoints/); + assert.equal(merged.newFindings, 0); + assert.equal(merged.run.persistedDedupInputs.filter((input) => input.dedupWorkerId === merged.id).length, 2); + assert.equal(counts.get("merge"), 2); + assert.equal(executions, 3); + assert.equal(run.workflowVersion, "deep-security-scan/v2"); + assert.deepEqual(JSON.parse(await readFile(merged.resultPath, "utf8")), merged.result); + assert.equal(merged.result.sourceCoverage.completeness, "complete"); + const snapshot = await store.get(run.scanId, "fixture-owner"); + const resumed = new DeepScanCoordinator({ + run: snapshot, store, pluginRoot: plugin, + executor: { run: async () => assert.fail("accepted recovery must not execute another model") }, + }); + const beforeRecovery = await readFile(merged.resultPath, "utf8"); + const recovered = await resumed.recoverAcceptedDiscoveries(); + assert.deepEqual(recovered.map(worker => worker.resultPath), [discovery.worker.resultPath, second.worker.resultPath]); + await rm(path.join(path.dirname(merged.resultPath), "..", "result.json")); + const reducers = await resumed.recoverCompletedReducers(recovered); + assert.equal(reducers.resultPath, merged.resultPath); + assert.deepEqual(reducers.result, merged.result); + assert.equal(await readFile(merged.resultPath, "utf8"), beforeRecovery, "recovery cannot rewrite accepted bytes"); + + } finally { + await rm(root, { recursive: true, force: true }); + } +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs new file mode 100644 index 000000000..1df47d525 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_checkpoint_coverage.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const continueAfterResume of [false, true]) { + test(`immutable discovery receipts survive resumed publication (continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-checkpoint-coverage-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume: true, continueAfterResume, immutableInputs: true, + }); + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"), "utf8")); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual(coverage.reviews.map((review) => review.completeness), ["partial", "complete", "unknown"]); + assert.equal(coverage.reviews[0].attempt, 2); + assert.equal(new Set(coverage.deferred.map((item) => item.candidateId)).size, 2); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const item of coverage.deferred) assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + for (const receipt of surface.receiptRefs) { + assert.equal(await readFile(path.join(scanDir, receipt), "utf8"), "Synthetic review evidence.\n"); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs new file mode 100644 index 000000000..d001192dd --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_compatibility.mjs @@ -0,0 +1,147 @@ +import assert from "node:assert/strict"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + entryPoints: [new URL("../src/deep-scan/registry.ts", import.meta.url).pathname], + format: "esm", + loader: { ".md": "text" }, + platform: "node", + write: false +}); +const { startOrJoinDeepScanCoordinator, DeepScanRemoteCoordinator } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); + +await testUnsupportedWorkflowDoesNotAcquireOwnership(); +await testUnsupportedSelectionDoesNotAcquireOwnership(); + +async function testUnsupportedSelectionDoesNotAcquireOwnership() { + for (const selection of [ + { workflowVersion: "deep-security-scan/v1", finalizationInput: { version: 1 } }, + { workflowVersion: "deep-security-scan/v2", finalizationInput: { version: 99 } } + ]) { + await assert.rejects(startOrJoinDeepScanCoordinator({ + begin: { run: { scanId: "fixture", schemaVersion: 1, ...selection }, shouldStart: false }, + registry: { + get: () => assert.fail("unsupported selection inspected a live coordinator"), + start: () => assert.fail("unsupported selection started a coordinator") + }, + options: { + threadId: "fixture-thread", + prepareExecutor: async () => assert.fail("unsupported selection resolved settings"), + store: { claimCoordinator: async () => assert.fail("unsupported selection acquired ownership") } + } + }), /finalization input version/); + } +} + +async function testUnsupportedWorkflowDoesNotAcquireOwnership() { + for (const version of [ + { schemaVersion: 99, workflowVersion: "deep-scan-mcp/v1" }, + { schemaVersion: 1, workflowVersion: "future/v99" } + ]) { + let mutations = 0; + await assert.rejects(startOrJoinDeepScanCoordinator({ + begin: { run: { scanId: "fixture", ...version }, shouldStart: false }, + registry: { + get: () => undefined, + start: () => { mutations += 1; } + }, + options: { + threadId: "fixture-thread", + store: { claimCoordinator: async () => { mutations += 1; } } + } + }), /unsupported workflow or schema version/); + assert.equal(mutations, 0); + } +} + + +// A joining client must not resolve or replace the live executor's settings. +for (const workflowVersion of ["deep-scan-mcp/v1", "deep-security-scan/v1", "deep-security-scan/v2"]) { + for (const local of [true, false]) { + let preparations = 0; + const run = { scanId: "fixture", status: "running", workflowVersion }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async () => { preparations += 1; return {}; }, + store: { claimCoordinator: async () => ({ run, acquired: false }) } + }; + await startOrJoinDeepScanCoordinator({ + begin: { run, shouldStart: false }, + registry: { get: () => local ? {} : undefined, start: () => assert.fail("observer started") }, + options + }); + assert.equal(preparations, 0); + } +} + +// Selected publication has no worker launch and must not need current settings. +for (const selected of [false, true]) { + const run = { + scanId: "fixture", status: "running", workflowVersion: "deep-security-scan/v2", + ...(selected ? { finalizationInput: { version: 1 } } : {}) + }; + let preparations = 0; + const fallback = {}; + const restored = {}; + await startOrJoinDeepScanCoordinator({ + begin: { run, shouldStart: false }, + registry: { + get: () => undefined, + start: (options) => { + assert.equal(options.executor, selected ? fallback : restored); + assert.equal(options.run, run); + return {}; + } + }, + options: { + threadId: "fixture-thread", + executor: fallback, + prepareExecutor: async () => { preparations += 1; return restored; }, + store: { claimCoordinator: async () => ({ run, acquired: true }) } + } + }); + assert.equal(preparations, selected ? 0 : 1); +} + +const originalNow = Date.now; +try { + let now = 0; + Date.now = () => now; + const run = { scanId: "fixture", status: "running", updatedAt: "1970-01-01T00:00:00Z" }; + const acquired = { ...run, model: "original-model", coordinatorGeneration: 3 }; + let preparations = 0; + const executor = { marker: "restored" }; + const registry = { + get: () => undefined, + start: (options) => { + assert.equal(options.run, acquired); + assert.equal(options.executor, executor); + return { wait: async () => ({ ...acquired, status: "succeeded" }) }; + } + }; + const options = { + threadId: "fixture-thread", + executor: { marker: "observer" }, + prepareExecutor: async (state) => { + assert.equal(state, acquired); + preparations += 1; + return executor; + }, + store: { + get: async () => run, + claimCoordinator: async () => ({ run: acquired, acquired: true }) + } + }; + await startOrJoinDeepScanCoordinator({ begin: { run, shouldStart: true }, registry, options }); + assert.equal(preparations, 1); + const remote = new DeepScanRemoteCoordinator({ run, registry, options }); + now = 60_000; + assert.equal((await remote.wait(undefined, 1_000)).status, "succeeded"); + assert.equal(preparations, 2, "takeover resolves settings from the newly acquired run"); +} finally { + Date.now = originalNow; +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs index 0625bbbb1..80a3e94db 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.mjs @@ -39,7 +39,8 @@ async function testCappedQueueAndSerialDedup() { retryDelaysMs: [1, 3, 9], clock: immediateClock, handoffClaimToken: "claim-fixture", - onComplete: async (draft) => completedDrafts.push(structuredClone(draft)) + onComplete: async (draft) => completedDrafts.push(structuredClone(draft)), + onFinalized: async () => assert.fail("Legacy scans retain explicit parent completion") }); coordinator.start(); const terminal = await coordinator.wait(undefined, 5_000); @@ -1362,12 +1363,13 @@ async function testFinishPersistenceFailureRewritesManifestAsFailure() { await assertFailureManifest(terminal, "terminal"); } -async function testLostFinishResponseReplaysWithoutOverwritingSuccessManifest() { +async function testLostFinishResponseObservesCommitWithoutOverwritingSuccessManifest() { const fixture = await fixtureRun({ workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 1 }); const store = new FakeStore(fixture.run); store.loseFirstFinishResponseAfterCommit = true; const coordinator = new DeepScanCoordinator({ run: fixture.run, + threadId: "original-parent-thread", store, executor: new FakeExecutor({ dedupNewFindings: [0] }), pluginRoot: fixture.pluginRoot, @@ -1377,36 +1379,12 @@ async function testLostFinishResponseReplaysWithoutOverwritingSuccessManifest() const terminal = await coordinator.wait(undefined, 5_000); assert.equal(terminal?.status, "succeeded"); - assert.equal(store.finishCalls.length, 2); - assert.deepEqual(store.finishCalls[1], store.finishCalls[0]); + assert.equal(store.finishCalls.length, 1); assert.equal(store.failCalls, 0); const manifest = JSON.parse(await readFile(terminal.manifestPath, "utf8")); assert.equal(manifest.scan.scanId, fixture.run.scanId); } -async function testLostWorkerCommitResponsesReplayIdempotently() { - const fixture = await fixtureRun({ workers: 1, subagents: 0, stopAfterNoNew: 1, maxDiscoveryRuns: 1 }); - const store = new FakeStore(fixture.run); - store.loseFirstDiscoveryAcceptanceResponseAfterCommit = true; - store.loseFirstDedupCommitResponseAfterCommit = true; - const coordinator = new DeepScanCoordinator({ - run: fixture.run, - store, - executor: new FakeExecutor({ dedupNewFindings: [0] }), - pluginRoot: fixture.pluginRoot, - clock: immediateClock - }); - coordinator.start(); - - const terminal = await coordinator.wait(undefined, 5_000); - assert.equal(terminal?.status, "succeeded"); - assert.equal(store.discoveryAcceptanceResponseLosses, 1); - assert.equal(store.dedupCommitResponseLosses, 1); - assert.equal(store.dedupCommitCalls.length, 2); - assert.equal(store.dedupCommits.length, 1); - assert.equal(store.failCalls, 0); -} - async function testCommittedReducerIsReconciledBeforeDiscoveryFailureManifest() { const fixture = await fixtureRun({ workers: 3, subagents: 0, stopAfterNoNew: 10, maxDiscoveryRuns: 3 }); const thirdWorkerGate = deferred(); @@ -2910,6 +2888,8 @@ async function testPausedDiscoverySurvivesCoordinatorRestart() { store.heartbeatCoordinator = async () => structuredClone(store.run); const replacementExecutor = new FakeExecutor({ dedupNewFindings: [0] }); const acceptedResult = await readFile(accepted.resultManifestPath, "utf8"); + const acceptedWorkerId = await workerIdFromPrompt(accepted.promptPath); + await Promise.all(persistedWorkers.map((worker) => rm(worker.promptPath, { force: true }))); const resumed = await startOrJoinDeepScanCoordinator({ begin: { run: structuredClone(store.run), shouldStart: false }, registry: new DeepScanCoordinatorRegistry(), @@ -2926,11 +2906,11 @@ async function testPausedDiscoverySurvivesCoordinatorRestart() { assert.equal(continuationClaims.length, 1); assert.equal(continuationClaims[0].handoffClaimToken, handoffClaimToken); - assert.equal(terminal?.status, "succeeded"); + assert.equal(terminal?.status, "succeeded", terminal?.error); assert.equal(store.failCalls, 0); assert.equal(replacementExecutor.logicalDiscoveryWorkers.size, 1); assert.equal( - replacementExecutor.logicalDiscoveryWorkers.has(await workerIdFromPrompt(accepted.promptPath)), + replacementExecutor.logicalDiscoveryWorkers.has(acceptedWorkerId), false ); assert.equal(store.dedupClaims.length, 1); @@ -3995,8 +3975,7 @@ try { await testConfigurationFailureDoesNotRetry(); await testFailureManifestWriteDoesNotMaskOriginalError(); await testFinishPersistenceFailureRewritesManifestAsFailure(); - await testLostFinishResponseReplaysWithoutOverwritingSuccessManifest(); - await testLostWorkerCommitResponsesReplayIdempotently(); + await testLostFinishResponseObservesCommitWithoutOverwritingSuccessManifest(); await testCommittedReducerIsReconciledBeforeDiscoveryFailureManifest(); await testLongWorkerErrorIsBoundedOnlyAtPersistenceBoundary(); await testDiscoveryPhasePersistenceFailureStopsDispatch(); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs new file mode 100644 index 000000000..00acf2808 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coverage_descriptions.mjs @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +const descriptions = { source: "source-review", worker: "focused-reviewer", task: "Check the filesystem boundary." }; +const sourceProvenance = { + ...descriptions, + workerId: "worker-local-label", attempt: 99, sourceId: "worker-local-source", candidateId: "worker-local-candidate", +}; + +for (const options of [ + {}, + { resume: true }, + { resume: true, continueAfterResume: true }, + { resume: true, selectedRecovery: true, discardMutableResults: true }, + { resume: true, selectedRecovery: true, discardMutableResults: true, splitSeededReducers: true }, +]) { + test(`coverage source descriptions survive publication and recovery: ${JSON.stringify(options)}`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-coverage-descriptions-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + ...options, immutableInputs: true, materialFindings: true, sourceProvenance, + }); + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"), "utf8")); + assert.equal(coverage.completeness, "partial"); + assert.equal(new Set(coverage.deferred.map((item) => item.candidateId)).size, 2); + for (const field of ["surfaces", "explicitExclusions", "deferred", "openQuestions"]) { + assert.ok(coverage[field].length > 0); + for (const item of coverage[field]) { + const review = coverage.reviews.find((review) => review.workerId === item.provenance.workerId); + assert.ok(review, "host identity names an actual independent review"); + assert.deepEqual(item.provenance, { + ...descriptions, workerId: review.workerId, attempt: review.attempt, + ...(field === "surfaces" ? { sourceId: "shared-surface" } : {}), + ...(field === "deferred" ? { sourceId: "same-id", candidateId: "candidate-1" } : {}), + }); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs index 76112891d..e1357d5c6 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs @@ -6,16 +6,18 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; +import { parse as parseToml } from "smol-toml"; const executorSource = new URL("../src/deep-scan/executor.ts", import.meta.url); const bundle = await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { "import.meta.url": JSON.stringify(executorSource.href) }, stdin: { // Test the environment snapshot without adding a production export. - contents: `${await readFile(executorSource, "utf8")}\nexport { snapshotWorkerEnvironment };`, + contents: `${await readFile(executorSource, "utf8")}\nexport { snapshotWorkerEnvironment };\nexport { captureDeepScanExecutionSettings, loadDeepScanExecutionSettings, restoredDeepScanWorkerSettings } from "./recovery-settings.js";\nexport { WorkbenchDeepScanStore } from "./store.js";`, loader: "ts", resolveDir: path.dirname(fileURLToPath(executorSource)), sourcefile: fileURLToPath(executorSource) @@ -24,7 +26,7 @@ const bundle = await build({ platform: "node", write: false }); -const { CodexSdkWorkerExecutor, resolveCodexPath, snapshotWorkerEnvironment } = await import( +const { WorkbenchDeepScanStore, CodexSdkWorkerExecutor, resolveCodexPath, snapshotWorkerEnvironment, captureDeepScanExecutionSettings, loadDeepScanExecutionSettings, restoredDeepScanWorkerSettings } = await import( `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` ); const errorsBundle = await build({ @@ -74,6 +76,9 @@ const deniedWorkerPermissionProfile = { try { await testOpenAiCredentialsReachWorker(); await testWorkerReasoningSummaries(); + await testWorkerProviderSelection(); + await testIsolatedReconstructedWorkers(); + if (process.platform !== "win32") await testNullUsageCompletion(); if (process.platform !== "win32") { await testMissingParentSandboxFailsBeforeWorkerLaunch(); await testDisallowedWorkerProfileFailsBeforeWorkerLaunch(); @@ -83,6 +88,7 @@ try { await testSdkInvocationAndThreadCapture(); await testBedrockCredentialsReachWorker(); await testArtifactServerUsesExtendedStartupTimeout(); + await testReducerCoveragePersistenceBinding(); await testZeroSubagentsPreservesHostRestrictions(); await testSdkResumesExistingThread(); await testRetryNotificationDoesNotInterruptTurn(); @@ -737,6 +743,293 @@ async function testOpenAiCredentialsReachWorker() { } } +async function testIsolatedReconstructedWorkers() { + const previousMarker = process.env.FAKE_CODEX_MARKER; + const originalSpawn = childProcess.spawn; + const scans = []; + try { + for (const name of ["first", "second"]) { + const uncappedSandbox = { filesystemDenies: trustedParentSandboxWithDenials.filesystemDenies }; + const currentParentSandbox = name === "first" ? uncappedSandbox : trustedParentSandboxWithDenials; + const expectedProfile = structuredClone(deniedWorkerPermissionProfile); + delete expectedProfile.filesystem.glob_scan_max_depth; + const fixture = await fakeCodexFixture(expectedProfile); + const codexHome = path.join(fixture.root, "home"); + const configPath = path.join(fixture.root, "scan config.toml"); + const promptPath = path.join(fixture.root, "prompt.md"); + await mkdir(codexHome); + const config = { + model: `fixture-${name}-inherited`, + model_provider: name === "first" ? "openrouter" : "amazon-bedrock", + model_reasoning_effort: "medium", + model_reasoning_summary: "concise", + service_tier: name === "first" ? "default" : "fast" + }; + await writeFile(configPath, Object.entries(config).filter(([key]) => name !== "first" + || !["model_provider", "model_reasoning_summary", "service_tier"].includes(key)) + .map(([key, value]) => `${key} = ${JSON.stringify(value)}\n`).join("") + + (name === "second" ? '[model_providers.amazon-bedrock.aws]\nregion = "us-west-2"\nprofile = "fixture-profile"\n' : "")); + const providerKeys = name === "first" ? ["OPENROUTER_API_KEY"] : ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]; + await writeFile(promptPath, "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE\n"); + const executable = path.join(fixture.root, process.platform === "win32" ? "node.exe" : "node"); + // Keep dynamically linked Node beside its libraries on Unix. Each scan + // still selects a distinct executable path at the spawn boundary. + if (process.platform === "win32") await copyFile(process.execPath, executable); + else await symlink(process.execPath, executable); + const codexOptions = { + codexPathOverride: executable, + baseUrl: `https://${name}.example.invalid/v1`, + env: { + PATH: path.dirname(process.execPath), + ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}), + CODEX_HOME: codexHome, + CODEX_SECURITY_CONFIG_PATH: configPath, + CODEX_API_KEY: `synthetic-${name}-credential`, + FAKE_CODEX_MARKER: fixture.markerPath, + FAKE_CODEX_PROVIDER_ENV_KEYS: JSON.stringify(providerKeys), + FAKE_CODEX_SCAN_VALUE: name + } + }; + const settings = { + codexOptions, + model: `fixture-${name}-override`, + reasoningEffort: "ultra", + usageOwner: { threadId: `fixture-${name}-owner`, turnId: "original-turn", startedAt: "2026-01-01T00:00:00Z" }, + parentSandbox: name === "first" ? trustedParentSandboxWithDenials : uncappedSandbox + }; + await mkdir(path.join(codexHome, "sessions")); + await writeFile(path.join(codexHome, "sessions", "owner.jsonl"), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", + payload: { id: `fixture-${name}-owner`, model_provider: config.model_provider } }, + { type: "event_msg", timestamp: "2026-01-01T00:00:00Z", + payload: { type: "thread_settings_applied", thread_id: `fixture-${name}-owner`, thread_settings: { + model: "native-parent-model", model_provider_id: config.model_provider, + reasoning_effort: "medium", reasoning_summary: config.model_reasoning_summary + } } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:01Z", + payload: { turn_id: "original-turn", model: "native-parent-model", effort: "medium", summary: "none" } }, + { type: "turn_context", timestamp: "2026-01-01T00:02:00Z", + payload: { turn_id: "later-turn", model: "later-parent-model", effort: "low", summary: "detailed" } } + ].map(JSON.stringify).join("\n") + "\n"); + await writeFile(path.join(codexHome, "sessions", "observer.jsonl"), JSON.stringify({ + type: "session_meta", timestamp: "2026-01-01T00:00:00Z", + payload: { id: `fixture-${name}-observer`, model_provider: "observer-provider" } + }) + "\n"); + const saved = await captureDeepScanExecutionSettings(settings, settings.parentSandbox, + { ...codexOptions.env, CODEX_CLI_PATH: executable }, + { threadId: `fixture-${name}-observer`, startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(saved.nativeServiceTierAbsent, name === "first" ? true : undefined); + const targetPath = path.join(fixture.root, "target"); + await mkdir(targetPath); + const workbenchPath = fileURLToPath(new URL("../../scripts/workbench_db.py", import.meta.url)); + const runWorkbench = async (args, input, _selectFinalization, withExecutionSettings) => { + const pythonArgs = withExecutionSettings ? ["-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](with_execution_settings=True)", + workbenchPath, ...args] : [workbenchPath, ...args]; + const result = spawnSync(process.env.PYTHON?.trim() || "python3", pythonArgs, { + env: { ...process.env, CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: path.join(fixture.root, "state") }, + input, encoding: "utf8", timeout: 30_000 + }); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + const store = new WorkbenchDeepScanStore(runWorkbench); + const beginInput = { targetPath, threadId: settings.usageOwner.threadId, + model: settings.model, reasoningEffort: settings.reasoningEffort, + scanRoot: path.join(fixture.root, "scans") }; + const { run } = await store.begin({ ...beginInput, executionSettings: saved }); + const recordedScanDir = run.scanDir; + const snapshotPath = path.join(recordedScanDir, "artifacts", "deep_discovery", "execution-settings.json"); + const snapshot = await readFile(snapshotPath, "utf8"); + const expectedProvider = name === "first" ? undefined + : { "amazon-bedrock": { aws: { region: "us-west-2", profile: "fixture-profile" } } }; + assert.deepEqual(JSON.parse(snapshot).settings.providerConfig, expectedProvider, + "actual workbench creation saves selections without catalog definitions"); + assert.deepEqual(await loadDeepScanExecutionSettings(recordedScanDir), saved); + const claim = await store.claimCoordinator({ scanId: run.scanId, threadId: beginInput.threadId }); + assert.equal(claim.acquired, true); + const observer = await new WorkbenchDeepScanStore(runWorkbench).begin({ + ...beginInput, executionSettings: null, model: "observer-model", reasoningEffort: "low" + }); + assert.equal(observer.shouldStart, false); + assert.equal(await readFile(snapshotPath, "utf8"), snapshot); + assert.equal(observer.run.model, settings.model); + assert.equal(snapshot.includes("synthetic-"), false); + const runtimeEnvironment = { ...codexOptions.env }; + const restored = restoredDeepScanWorkerSettings(saved, currentParentSandbox, () => runtimeEnvironment); + restored.codexOptions.baseUrl = codexOptions.baseUrl; + scans.push({ name, fixture, recordedScanDir, currentParentSandbox, config, configPath, promptPath, settings, runtimeEnvironment, snapshotPath, snapshot, providerKeys, expectedProvider, + executor: new CodexSdkWorkerExecutor(restored) }); + } + childProcess.spawn = (command, args, options) => { + const scan = scans.find((scan) => options?.env?.FAKE_CODEX_MARKER === scan.fixture.markerPath); + if (scan) { + const configured = scan.settings.codexOptions.codexPathOverride; + assert.ok(command === configured || command === path.toNamespacedPath(configured)); + } + return originalSpawn(command, scan ? [scan.fixture.executablePath, ...args] : args, options); + }; + syncBuiltinESMExports(); + + for (const phase of ["fresh", "resume", "reconstructed", "incomplete"]) { + if (phase === "reconstructed" || phase === "incomplete") { + for (const scan of scans) { + // The caller restores recorded selections. Its old config file need + // not exist; current credentials still come from the selected home/env. + if (phase === "reconstructed") await rm(scan.configPath); + if (phase === "incomplete") { + const saved = JSON.parse(scan.snapshot); + for (const key of ["model", "reasoningEffort", "reasoningSummary"]) delete saved.settings[key]; + // Native history restores the first provider. The second snapshot + // retains the provider binding for its saved AWS selectors. + if (scan.name === "first") delete saved.settings.modelProvider; + if (scan.name === "first") delete saved.settings.serviceTier; + await writeFile(scan.snapshotPath, JSON.stringify(saved)); + } + const snapshotBeforeRead = await readFile(scan.snapshotPath, "utf8"); + const recorded = await loadDeepScanExecutionSettings(scan.recordedScanDir, { + ...scan.settings, createdAt: "2026-01-01T00:01:00Z" + }); + const restored = restoredDeepScanWorkerSettings(recorded, scan.currentParentSandbox, () => scan.runtimeEnvironment); + restored.codexOptions.baseUrl = scan.settings.codexOptions.baseUrl; + scan.executor = new CodexSdkWorkerExecutor(restored); + assert.equal(await readFile(scan.snapshotPath, "utf8"), snapshotBeforeRead, + "restoring original worker selections must not rewrite saved settings"); + } + } + for (const scan of scans) { + scan.runtimeEnvironment.CODEX_API_KEY = `synthetic-${scan.name}-${phase}`; + for (const key of scan.providerKeys) scan.runtimeEnvironment[key] = `synthetic-${scan.name}-${phase}-${key}`; + scan.runtimeEnvironment.FAKE_CODEX_SCAN_VALUE = `${scan.name}-${phase}`; + scan.runtimeEnvironment.CODEX_HOME = path.join(scan.fixture.root, "observer-home"); + scan.runtimeEnvironment.CODEX_CLI_PATH = path.join(scan.fixture.root, "observer-codex"); + } + for (const kind of ["discovery", "dedup"]) { + await Promise.all(scans.map(async (scan) => { + const resumeThreadId = phase === "fresh" ? undefined : `fixture-${scan.name}-resumed`; + const result = await scan.executor.run({ + kind, promptPath: scan.promptPath, workingDirectory: scan.fixture.root, + subagents: scan.name === "first" ? 0 : 2, + resumeThreadId, continuationPrompt: "CAPTURE_SYNTHETIC_OPENAI_AUTH CAPTURE_SYNTHETIC_PROVIDER_AUTH NULL_USAGE continuation", + signal: new AbortController().signal + }); + assert.equal(result.threadId, resumeThreadId ?? "fixture-thread-id"); + const child = JSON.parse(await readFile(scan.fixture.markerPath, "utf8")); + const preflight = JSON.parse(await readFile(scan.fixture.preflightMarkerPath, "utf8")); + assert.equal(await realpath(child.executable), await realpath(scan.settings.codexOptions.codexPathOverride)); + assert.equal(child.codexCliPath, scan.settings.codexOptions.codexPathOverride); + assert.equal(child.codexHome, scan.settings.codexOptions.env.CODEX_HOME); + assert.equal(preflight.codexHome, child.codexHome); + assert.equal(child.scanValue, `${scan.name}-${phase}`); + assert.equal(child.configPath, scan.configPath); + assert.deepEqual(child.openaiAuthentication, { CODEX_API_KEY: `synthetic-${scan.name}-${phase}` }); + assert.deepEqual(child.providerAuthentication, Object.fromEntries(scan.providerKeys + .map((key) => [key, `synthetic-${scan.name}-${phase}-${key}`]))); + assertFlagPair(child.argv, "--model", scan.settings.model); + for (const key of ["model_provider", "model_reasoning_summary", "service_tier"]) { + const override = `${key}=${JSON.stringify(scan.config[key])}`; + assert.equal(child.argv.includes(override), true, override); + assert.equal(preflight.argv.includes(override), true, override); + } + assert.equal(child.argv.includes('model_reasoning_effort="ultra"'), true); + assert.equal(preflight.argv.includes('model_reasoning_effort="ultra"'), true); + assert.equal(preflight.argv.includes(`model=${JSON.stringify(scan.settings.model)}`), true); + const baseUrl = `openai_base_url=${JSON.stringify(scan.settings.codexOptions.baseUrl)}`; + assert.equal(child.argv.includes(baseUrl), true); + assert.equal(preflight.argv.includes(baseUrl), true); + for (const launch of [child, preflight]) { + const provider = launch.argv.filter((argument) => /^model_providers[.=]/u.test(argument)); + assert.ok(provider.length > 0, "both preflight and worker launch receive provider configuration"); + const providers = parseToml(provider.join("\n")).model_providers; + if (scan.expectedProvider) assert.deepEqual(providers, scan.expectedProvider); + else assert.deepEqual(Object.keys(providers.openrouter).sort(), ["base_url", "env_key", "name", "wire_api"]); + assert.equal(workerPermissionProfileOverride(launch.argv).includes("glob_scan_max_depth"), false, + "a resumed bounded cap must not truncate an original uncapped deny glob, in either order"); + } + assertReadOnlyWorkerPolicy(child.argv); + assertWorkerSubagentPolicy(child.argv, scan.name === "first" ? 0 : 2); + assert.equal(workerPermissionProfileOverride(child.argv).includes('"/repo/.env"="deny"'), true); + assert.equal(child.argv.includes("resume"), resumeThreadId !== undefined); + assert.equal(child.stdin.includes("continuation"), resumeThreadId !== undefined); + })); + } + if (phase === "fresh") { + for (const scan of scans) { + await writeFile(scan.configPath, 'model_provider = "changed-provider"\nmodel_reasoning_summary = "detailed"\n'); + } + } + } + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} + +async function testWorkerProviderSelection() { + const fixture = await fakeCodexFixture(); + const saved = Object.fromEntries( + ["CODEX_CLI_PATH", "CODEX_SECURITY_CONFIG_PATH", "OPENAI_API_KEY", "CODEX_API_KEY"].map((name) => [name, process.env[name]]) + ); + const originalSpawn = childProcess.spawn; + try { + delete process.env.OPENAI_API_KEY; + delete process.env.CODEX_API_KEY; + const configPath = path.join(fixture.root, "scan config.toml"); + const promptPath = path.join(fixture.root, "prompt.md"); + await writeFile(configPath, 'model_provider = "fixture-provider"\n'); + await writeFile(promptPath, "fixture provider selection"); + process.env.CODEX_CLI_PATH = process.execPath; + process.env.CODEX_SECURITY_CONFIG_PATH = configPath; + childProcess.spawn = (command, args, options) => originalSpawn( + command, + command === process.execPath || command === path.toNamespacedPath(process.execPath) + ? [fixture.executablePath, ...args] + : args, + options + ); + syncBuiltinESMExports(); + const executor = new CodexSdkWorkerExecutor({ parentSandbox: trustedParentSandbox }); + for (const kind of ["discovery", "dedup"]) { + await executor.run({ + kind, promptPath, workingDirectory: fixture.root, subagents: 0, + signal: new AbortController().signal + }); + const invocation = JSON.parse(await readFile(fixture.markerPath, "utf8")); + assert.equal(invocation.argv.includes('model_provider="fixture-provider"'), true); + } + } finally { + childProcess.spawn = originalSpawn; + syncBuiltinESMExports(); + for (const [name, value] of Object.entries(saved)) restoreEnv(name, value); + } +} + +async function testNullUsageCompletion() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + await writeFile(promptPath, "NULL_USAGE\n"); + for (const kind of ["discovery", "dedup"]) { + for (const resumeThreadId of [undefined, "fixture-resumed-thread"]) { + const result = await new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox + }).run({ + kind, promptPath, workingDirectory: fixture.root, subagents: 0, + resumeThreadId, signal: new AbortController().signal + }); + assert.equal(result.threadId, resumeThreadId ?? "fixture-thread-id"); + assert.equal(result.finalResponse, "fixture final response"); + } + } + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + } +} + async function testWorkerReasoningSummaries() { const cases = [ ["", undefined], @@ -922,6 +1215,56 @@ async function testArtifactServerUsesExtendedStartupTimeout() { } } +async function testReducerCoveragePersistenceBinding() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + const previousMarker = process.env.FAKE_CODEX_MARKER; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + await writeFile(promptPath, "fixture reducer prompt\n"); + const launches = []; + for (const resume of [false, true]) { + for (const persistSourceCoverage of [false, true]) { + const markerPath = path.join(fixture.root, `coverage-${resume}-${persistSourceCoverage}.json`); + process.env.FAKE_CODEX_MARKER = markerPath; + const scanRoot = path.join(fixture.root, `scan-${resume}-${persistSourceCoverage}`); + const deepReducer = { + scanRoot, + claimedWorkers: [{ + id: "worker-1", attempt: 2, + resultPath: path.join(scanRoot, "worker", "checkpoints", "accepted.json"), + artifactDir: path.join(scanRoot, "worker"), + }], + persistSourceCoverage, + }; + const launch = new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox, + artifactContext: { pluginRoot: fixture.root, scanRoot: deepReducer.scanRoot, repoRoot: fixture.root, scanId: "fixture-scan-id" }, + }).run({ + kind: "dedup", promptPath, workingDirectory, subagents: 0, + signal: new AbortController().signal, + ...(resume ? { resumeThreadId: "fixture-existing-thread", continuationPrompt: "continue the reducer\n" } : {}), + artifactContext: { root: workingDirectory, layout: "reducer", deepReducer }, + }); + launches.push(launch.then(async () => { + const invocation = JSON.parse(await readFile(markerPath, "utf8")); + const prefix = "mcp_servers.cs_artifacts.env.CODEX_SECURITY_REDUCER_CONTEXT_JSON="; + const encoded = invocation.argv.find((arg) => arg.startsWith(prefix)); + assert.ok(encoded, "the launched reducer receives its host-bound artifact context"); + assert.deepEqual(JSON.parse(JSON.parse(encoded.slice(prefix.length))), deepReducer); + })); + } + } + await Promise.all(launches); + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + restoreEnv("FAKE_CODEX_MARKER", previousMarker); + } +} + async function testSdkResumesExistingThread() { const fixture = await fakeCodexFixture(); const previousPath = process.env.CODEX_CLI_PATH; @@ -1400,22 +1743,27 @@ async function testDisallowedWorkerProfileFailsBeforeWorkerLaunch() { await mkdir(workingDirectory); await writeFile(promptPath, "fixture blocked worker prompt\n"); - await assert.rejects( - new CodexSdkWorkerExecutor({ - parentSandbox: trustedParentSandbox - }).run({ - kind: "discovery", - promptPath, - workingDirectory, - subagents: 0, - signal: new AbortController().signal - }), - (error) => error?.name === "DeepScanNonRetryableError" - && error.message.includes("codex_security_deep_scan_worker") - && error.message.includes("[allowed_permission_profiles]") - && error.message.includes("codex_security_deep_scan_worker = true") - && error.message.includes("Deep Scan did not run.") - ); + for (const kind of ["discovery", "dedup"]) { + for (const resumeThreadId of [undefined, "fixture-resumed-worker"]) { + await assert.rejects( + new CodexSdkWorkerExecutor({ + parentSandbox: trustedParentSandbox + }).run({ + kind, + resumeThreadId, + promptPath, + workingDirectory, + subagents: 0, + signal: new AbortController().signal + }), + (error) => error?.name === "DeepScanNonRetryableError" + && error.message.includes("codex_security_deep_scan_worker") + && error.message.includes("[allowed_permission_profiles]") + && error.message.includes("codex_security_deep_scan_worker = true") + && error.message.includes("Deep Scan did not run.") + ); + } + } await assert.rejects( readFile(fixture.markerPath, "utf8"), (error) => error?.code === "ENOENT" @@ -1480,7 +1828,7 @@ async function fakeCodexFixture( `const accountResult = ${JSON.stringify(accountResult)};`, `const preflightMarkerPath = ${JSON.stringify(preflightMarkerPath)};`, "if (process.argv.includes('app-server')) {", - " const preflight = { cwd: process.cwd(), codexHome: process.env.CODEX_HOME, requests: [] };", + " const preflight = { argv: process.argv.slice(2), cwd: process.cwd(), codexHome: process.env.CODEX_HOME, requests: [] };", " writeFileSync(preflightMarkerPath, JSON.stringify(preflight));", " let buffer = '';", " process.stdin.setEncoding('utf8');", @@ -1522,7 +1870,8 @@ async function fakeCodexFixture( "for await (const chunk of process.stdin) stdin += chunk;", "const openaiAuthentication = stdin.includes('CAPTURE_SYNTHETIC_OPENAI_AUTH') ? { OPENAI_API_KEY: process.env.OPENAI_API_KEY, CODEX_API_KEY: process.env.CODEX_API_KEY } : undefined;", "const bedrockAuthentication = stdin.includes('CAPTURE_SYNTHETIC_BEDROCK_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_BEDROCK_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined;", - "writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ argv: process.argv.slice(2), stdin, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, configPath: process.env.CODEX_SECURITY_CONFIG_PATH, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}) }));", + "const providerAuthentication = stdin.includes('CAPTURE_SYNTHETIC_PROVIDER_AUTH') ? Object.fromEntries(JSON.parse(process.env.FAKE_CODEX_PROVIDER_ENV_KEYS).map((name) => [name, process.env[name]])) : undefined;", + "writeFileSync(process.env.FAKE_CODEX_MARKER, JSON.stringify({ executable: process.execPath, argv: process.argv.slice(2), stdin, cwd: process.cwd(), codexHome: process.env.CODEX_HOME, codexCliPath: process.env.CODEX_CLI_PATH, configPath: process.env.CODEX_SECURITY_CONFIG_PATH, scanValue: process.env.FAKE_CODEX_SCAN_VALUE, deepConfigPath: process.env.CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH, originator: process.env.CODEX_INTERNAL_ORIGINATOR_OVERRIDE, ...(stdin.includes('COMPLETE_THEN_HANG') ? { pid: process.pid } : {}), ...(openaiAuthentication ? { openaiAuthentication } : {}), ...(bedrockAuthentication ? { bedrockAuthentication } : {}), ...(providerAuthentication ? { providerAuthentication } : {}) }));", "if (stdin.includes('COMPLETE_THEN_HANG')) process.on('SIGTERM', () => setTimeout(() => process.exit(0), 100));", "if (stdin.includes('THREAD_START_CONFIG_ERROR')) { console.error('Error: thread/start: thread/start failed: agents.max_threads cannot be set when features.multi_agent_v2 is enabled (code -32600)'); process.exit(1); }", "if (stdin.includes('CONFIG_ERROR')) { console.error('failed to load configuration: invalid value'); process.exit(2); }", @@ -1562,7 +1911,7 @@ async function fakeCodexFixture( " console.log(JSON.stringify({ type: 'item.completed', item }));", "}", "console.log(JSON.stringify({ type: 'item.completed', item: { id: 'message-1', type: 'agent_message', text: 'fixture final response' } }));", - "console.log(JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 1, cached_input_tokens: 0, output_tokens: 1 } }));", + "console.log(JSON.stringify({ type: 'turn.completed', usage: stdin.includes('NULL_USAGE') ? null : { input_tokens: 1, cached_input_tokens: 0, output_tokens: 1 } }));", "if (stdin.includes('COMPLETE_THEN_HANG')) { setInterval(() => {}, 1_000); await new Promise(() => {}); }", "}", "" diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs new file mode 100644 index 000000000..8d725ef84 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_finalization.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + stdin: { + contents: `export * from "./src/deep-scan/finalization.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "./src/artifact-scan-draft.ts";`, + resolveDir: path.resolve(import.meta.dirname, ".."), + }, + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=deep-scan-finalization-contract.js" }, +}); +const { publishSelectedDeepScan, readSelectedDeepScanDraft, createDeepScanArtifacts, saveScanDraftCheckpoint } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`, +); +const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + +for (const terminalReason of ["capped", "saturated"]) { + test(`replays the selected ${terminalReason} input after replaceable results change`, async () => { + const scanDir = await realpath(await mkdtemp(path.join(tmpdir(), "selected-finalization-"))); + try { + const artifacts = createDeepScanArtifacts(scanDir); + const root = path.join(artifacts.dedupRoot, "dedup-0001", "output"); + await mkdir(root, { recursive: true }); + const draft = { + scanId, complete: true, findings: [], + coverage: { + completeness: "partial", surfaces: [], explicitExclusions: [], + deferred: [{ id: "review", reason: "A dependency remains unreviewed." }], + }, + }; + const { coverage, ...reduction } = draft; + await saveScanDraftCheckpoint({ root, repoRoot: scanDir, layout: "reducer" }, { + ...reduction, sourceCoverage: coverage, + }); + const [name] = await readdir(path.join(root, "checkpoints")); + const checkpoint = path.join(root, "checkpoints", name); + const contents = await readFile(checkpoint); + const selection = { + version: 1, + resultPath: path.relative(scanDir, checkpoint), + resultSha256: createHash("sha256").update(contents).digest("hex"), + terminalReason, omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + }; + // A replacement result is not a new finalization selection. + await writeFile(path.join(root, "result.json"), JSON.stringify({ ...draft, complete: false })); + assert.deepEqual(await readSelectedDeepScanDraft(artifacts, scanId, selection), draft); + assert.deepEqual(await readSelectedDeepScanDraft(artifacts, scanId, selection), draft); + await assert.rejects( + readSelectedDeepScanDraft(artifacts, "e14e9229-653a-4385-bec0-8745f0b037cb", selection), + /complete result for this scan/, + ); + await writeFile(checkpoint, JSON.stringify({ ...draft, findings: [] })); + await assert.rejects(readSelectedDeepScanDraft(artifacts, scanId, selection), /changed after acceptance/); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} + +test("recreates only partial coverage for a persisted zero-success deadline selection", async () => { + const selection = { + version: 1, resultPath: null, resultSha256: null, terminalReason: "capped", + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + }; + const result = await readSelectedDeepScanDraft(createDeepScanArtifacts("unused"), scanId, selection); + assert.equal(result.scanId, scanId); + assert.deepEqual(result.findings, []); + assert.equal(result.coverage.completeness, "partial"); + assert.equal(result.coverage.deferred.length, 1); + await assert.rejects( + readSelectedDeepScanDraft(createDeepScanArtifacts("unused"), scanId, { ...selection, terminalReason: "saturated" }), + /recorded discovery deadline/, + ); +}); + +for (const status of ["failed", "canceled", "interrupted"]) { + test(`saved selection does not turn a ${status} scan into success`, async () => { + await assert.rejects(publishSelectedDeepScan({ + run: { scanId, scanDir: "unused", workflowVersion: "deep-security-scan/v2", status, finalizationInput: { + version: 1, resultPath: null, resultSha256: null, terminalReason: "capped", + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + } }, + artifacts: createDeepScanArtifacts("unused"), signal: new AbortController().signal, + publish: async () => assert.fail("Stopped work cannot publish successful results"), + finish: async () => assert.fail("Stopped work cannot finish successfully"), + }), /Stopped Deep Scan/); + }); +} + +test("cancellation prevents selected publication and preserves its input", async () => { + const controller = new AbortController(); + controller.abort("cost limit or user cancellation"); + const selection = { version: 1, resultPath: null, resultSha256: null, terminalReason: "capped", + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z" }; + await assert.rejects(publishSelectedDeepScan({ + run: { scanId, scanDir: "unused", workflowVersion: "deep-security-scan/v2", status: "running", finalizationInput: selection }, + artifacts: createDeepScanArtifacts("unused"), signal: controller.signal, + publish: async () => assert.fail("Canceled work cannot publish"), + finish: async () => assert.fail("Canceled work cannot finish"), + }), (error) => error === controller.signal.reason); + assert.equal(selection.terminalReason, "capped"); +}); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs new file mode 100644 index 000000000..71ea1af4f --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_material_coverage.mjs @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { materialRemediations, materialRemediationTests, publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const [resume, continueAfterResume] of [[false, false], [true, false], [true, true]]) { + test(`material fixes and unresolved coverage survive canonical publication (resume: ${resume}, continued: ${continueAfterResume})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "deep-material-coverage-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume, continueAfterResume, immutableInputs: true, materialFindings: true, + }); + const { findings } = JSON.parse(await readFile(path.join(scanDir, "findings.json"), "utf8")); + assert.equal(findings.length, 1); + const sources = findings[0].provenance.sourceFindings; + assert.deepEqual(sources.map((source) => source.finding.remediation), materialRemediations); + assert.equal(new Set(sources.map((source) => source.id)).size, 2); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const text of [...materialRemediations, ...materialRemediationTests]) { + assert.equal(report.split(text).length - 1, 1, `canonical report retains ${text}`); + } + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"), "utf8")); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual(coverage.reviews.map((review) => review.completeness), ["partial", "complete", "unknown"]); + assert.equal(coverage.deferred.length, 2, "a finding does not discharge independent unresolved work"); + assert.equal(new Set(coverage.deferred.map((item) => item.candidateId)).size, 2); + for (const item of coverage.deferred) assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + assert.equal(await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), "Synthetic review evidence.\n"); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs new file mode 100644 index 000000000..8a7693dc6 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_recorded_recovery.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { materialRemediations, materialRemediationTests, publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const [continueAfterResume, legacyAttempts, splitSeededReducers] of [[false, false, false], [true, false, false], [true, true, false], [false, false, true], [false, true, true]]) { + test(`recorded inputs preserve fixes and coverage after mutable outputs disappear (continued: ${continueAfterResume}, legacy attempts: ${legacyAttempts}, reducer chain: ${splitSeededReducers})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "recorded-coverage-recovery-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir } = await publishCoverageFixture(fixture, "partial", { + resume: true, continueAfterResume, legacyAttempts, splitSeededReducers, + immutableInputs: true, materialFindings: true, discardMutableResults: true, + }); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const fix of [...materialRemediations, ...materialRemediationTests]) { + assert.equal(report.split(fix).length - 1, 1); + } + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"), "utf8")); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual(coverage.reviews.map((review) => review.completeness), ["partial", "complete", "unknown"]); + assert.equal(new Set(coverage.deferred.map((item) => item.candidateId)).size, 2); + for (const surface of coverage.surfaces) { + assert.equal(await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), "Synthetic review evidence.\n"); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs new file mode 100644 index 000000000..a32622d21 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_recovery_settings.mjs @@ -0,0 +1,369 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; + +const bundle = await build({ + bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], + define: { "import.meta.url": JSON.stringify(new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).href) }, + entryPoints: [new URL("../src/deep-scan/recovery-settings.ts", import.meta.url).pathname], + format: "esm", + platform: "node", + write: false +}); +const { captureDeepScanExecutionSettings: captureSettings, restoredDeepScanWorkerSettings: restoreSettings, loadDeepScanExecutionSettings: loadSettings } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}` +); +const root = await mkdtemp(join(tmpdir(), "deep-settings-")); +try { + const settings = { + codexPath: "/fixture/runtime/codex", + codexHome: "/fixture/account", + model: "fixture-model", + modelProvider: "fixture-provider", + reasoningEffort: "high", + reasoningSummary: "detailed", + serviceTier: "fast" + }; + const globSandbox = (depth) => ({ + filesystemDenies: ["/fixture/**/*.secret"], + ...(depth === undefined ? {} : { globScanMaxDepth: depth }) + }); + for (const [originalDepth, currentDepth, expectedDepth] of [ + [2, 5, 5], [5, 2, 5], [undefined, 2, undefined], [2, undefined, undefined] + ]) { + const restored = restoreSettings({ ...settings, parentSandbox: globSandbox(originalDepth) }, + globSandbox(currentDepth)); + assert.equal(restored.parentSandbox.globScanMaxDepth, expectedDepth, + `deny expansion must preserve both policies: ${originalDepth}, ${currentDepth}`); + } + assert.equal(restoreSettings(settings, globSandbox(2)).parentSandbox.globScanMaxDepth, 2, + "unavailable historical policy does not establish uncapped glob expansion"); + assert.equal(restoreSettings({ ...settings, parentSandbox: globSandbox(2) }, { + filesystemDenies: ["/fixture/exact-denial"] + }).parentSandbox.globScanMaxDepth, 2, "exact denials do not change glob expansion"); + const writeSnapshot = async (directory, value) => { + const path = join(directory, "artifacts", "deep_discovery", "execution-settings.json"); + await mkdir(join(directory, "artifacts", "deep_discovery"), { recursive: true }); + await writeFile(path, JSON.stringify({ version: 1, settings: value }, null, 2) + "\n"); + }; + await assert.rejects(loadSettings(join(root, "missing")), /no recorded original execution settings/); + await writeSnapshot(join(root, "one"), settings); + await writeSnapshot(join(root, "two"), { ...settings, model: "other-model" }); + const savedPath = join(root, "one", "artifacts", "deep_discovery", "execution-settings.json"); + const saved = await readFile(savedPath, "utf8"); + const [recovered, concurrent] = await Promise.all([ + loadSettings(join(root, "one")), loadSettings(join(root, "two")) + ]); + assert.deepEqual(recovered, settings); + assert.equal(concurrent.model, "other-model"); + assert.equal(await readFile(savedPath, "utf8"), saved); + recovered.model = "caller-mutation"; + assert.deepEqual(await loadSettings(join(root, "one")), settings); + const configPath = join(root, "runtime.toml"); + await writeFile(configPath, `model = "inherited-model" +model_provider = "custom" +profile = "scan" +[profiles.scan] +model_reasoning_summary = "concise" +service_tier = "flex" +[model_providers.custom] +name = "Fixture" +http_headers = { Authorization = "synthetic-secret" } +`); + const captured = await captureSettings({ model: "original-model", reasoningEffort: "ultra" }, { + filesystemDenies: ["/fixture/original-deny"], globScanMaxDepth: 3 + }, { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root, CODEX_SECURITY_CONFIG_PATH: configPath }); + assert.equal(captured.model, "original-model"); + assert.equal(captured.modelProvider, "custom"); + assert.equal(captured.reasoningSummary, "concise"); + assert.equal(captured.serviceTier, "flex"); + assert.equal(captured.providerConfig, undefined); + assert.equal(JSON.stringify(captured).includes("synthetic-secret"), false); + for (const modelProvider of ["openrouter", "fireworks", "amazon-bedrock"]) { + await writeFile(configPath, `model_provider = ${JSON.stringify(modelProvider)} +[model_providers.${modelProvider}.aws] +region = "us-west-2" +profile = "fixture-profile" +access_key_id = "synthetic-secret" +`); + const selected = await captureSettings({}, { filesystemDenies: [] }, { + CODEX_CLI_PATH: process.execPath, CODEX_HOME: root, CODEX_SECURITY_CONFIG_PATH: configPath + }); + assert.equal(selected.modelProvider, modelProvider); + const expectedProvider = modelProvider === "amazon-bedrock" + ? { "amazon-bedrock": { aws: { region: "us-west-2", profile: "fixture-profile" } } } : undefined; + assert.deepEqual(selected.providerConfig, expectedProvider, + "saved selections exclude catalog definitions and retain Bedrock selectors"); + const providerDir = join(root, modelProvider); + await writeSnapshot(providerDir, selected); + const path = join(providerDir, "artifacts", "deep_discovery", "execution-settings.json"); + const bytes = await readFile(path, "utf8"); + assert.equal(bytes.includes("synthetic-secret"), false); + const restoredProvider = restoreSettings(await loadSettings(providerDir), { filesystemDenies: [] }) + .codexOptions.config.model_providers; + if (expectedProvider) assert.deepEqual(restoredProvider, expectedProvider); + else assert.deepEqual(Object.keys(restoredProvider[modelProvider]).sort(), + ["base_url", "env_key", "name", "wire_api"]); + assert.equal(await readFile(path, "utf8"), bytes); + // Older snapshots can contain catalog definitions. Reading them must not + // rewrite their bytes or prevent the existing launch projection. + if (!expectedProvider) { + await writeSnapshot(providerDir, { ...selected, providerConfig: restoredProvider }); + const legacyBytes = await readFile(path, "utf8"); + const legacy = await loadSettings(providerDir); + assert.equal(legacy.providerConfig, undefined); + assert.deepEqual(restoreSettings(legacy, { filesystemDenies: [] }).codexOptions.config.model_providers, + restoredProvider); + assert.equal(await readFile(path, "utf8"), legacyBytes); + } + } + let credential = "synthetic-first"; + const restored = restoreSettings(captured, { filesystemDenies: ["/fixture/current-deny"] }, () => ({ + CODEX_API_KEY: credential, CODEX_HOME: "/fixture/observer-home", CODEX_CLI_PATH: "/fixture/observer-codex" + })); + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-first"); + credential = "synthetic-refreshed"; + assert.equal(restored.codexOptions.env.CODEX_API_KEY, "synthetic-refreshed"); + assert.equal(restored.codexOptions.env.CODEX_HOME, root); + assert.equal(restored.codexOptions.env.CODEX_CLI_PATH, captured.codexPath); + assert.deepEqual(restored.parentSandbox.filesystemDenies, ["/fixture/original-deny", "/fixture/current-deny"]); + assert.equal(restored.codexOptions.config.model_reasoning_effort, "ultra"); + await writeFile(join(root, "config.toml"), 'model = "native-home-model"\n'); + const native = await captureSettings({}, { filesystemDenies: [] }, { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root }); + assert.equal(native.model, "native-home-model"); + const sessionDirectory = join(root, "sessions"); + await mkdir(sessionDirectory); + await writeFile(join(sessionDirectory, "parent.jsonl"), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", payload: { id: "fixture-parent", model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:01Z", payload: { turn_id: "original-turn", model: "parent-model", effort: "high", summary: "none" } }, + { type: "turn_context", timestamp: "2026-01-01T00:02:00Z", payload: { turn_id: "later-turn", model: "later-model", effort: "low", summary: "detailed" } } + ].map(JSON.stringify).join("\n") + "\n"); + await writeFile(join(sessionDirectory, "other.jsonl"), JSON.stringify({ + type: "session_meta", payload: { id: "fixture-other", model_provider: "other-provider" } + }) + "\n"); + const parentSettings = await captureSettings({}, { filesystemDenies: [] }, { + CODEX_CLI_PATH: process.execPath, CODEX_HOME: root + }, { threadId: "fixture-parent", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(parentSettings.model, "native-home-model", "explicit config retains precedence"); + assert.equal(parentSettings.modelProvider, "openai"); + assert.equal(parentSettings.reasoningSummary, "none", "later owner turns are not original discovery settings"); + assert.equal(parentSettings.reasoningEffort, "high"); + await writeFile(join(root, "config.toml"), ""); + const parentEnvironment = { CODEX_CLI_PATH: process.execPath, CODEX_HOME: root }; + const [originalParent, otherParent, unavailableParent] = await Promise.all([ + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-parent", startedAt: "2026-01-01T00:00:01Z" }), + captureSettings({}, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:00:01Z" }), + captureSettings({ model: "stored-model", reasoningEffort: "ultra" }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-unavailable", startedAt: "2026-01-01T00:00:01Z" }) + ]); + assert.equal(originalParent.model, "parent-model"); + assert.equal(originalParent.reasoningSummary, "none", "the original turn is included at its timestamp"); + assert.equal(otherParent.modelProvider, "other-provider"); + assert.equal(otherParent.model, undefined, "concurrent scans do not borrow another parent's model"); + assert.equal(otherParent.reasoningSummary, undefined); + assert.equal(unavailableParent.model, "stored-model"); + assert.equal(unavailableParent.reasoningEffort, "ultra"); + assert.equal(unavailableParent.modelProvider, undefined, "missing history does not establish a provider"); + assert.equal(unavailableParent.reasoningSummary, undefined); + assert.equal(unavailableParent.serviceTier, undefined); + assert.equal(unavailableParent.nativeServiceTierAbsent, undefined, "missing history does not prove native absence"); + const originalOwner = { threadId: "fixture-parent", turnId: "original-turn", startedAt: "2026-01-01T00:00:00Z" }; + for (const workflowVersion of ["deep-security-scan/v1", "deep-scan-mcp/v1"]) { + const legacyDir = join(root, workflowVersion.replaceAll("/", "-")); + const legacy = await loadSettings(legacyDir, { + workflowVersion, model: "recorded-model", reasoningEffort: "ultra", + createdAt: "2026-01-01T00:01:00Z", usageOwner: null + }, async () => ({ config: { model_reasoning_summary: "concise", service_tier: "flex" }, + usageOwner: originalOwner }), parentEnvironment); + assert.equal(legacy.model, "recorded-model"); + assert.equal(legacy.reasoningSummary, "concise", "recorded recipe retains precedence"); + assert.equal(legacy.modelProvider, "openai", "recorded original owner supplies native selections"); + assert.equal(legacy.serviceTier, "flex"); + assert.equal(legacy.codexPath, undefined, "legacy metadata did not record an executable"); + assert.equal(legacy.codexHome, undefined, "a history lookup home is not recorded execution provenance"); + const restoredLegacy = restoreSettings(legacy, { filesystemDenies: ["/fixture/current-deny"] }, + () => ({ CODEX_HOME: "/fixture/runtime-home", CODEX_CLI_PATH: "/fixture/runtime-codex", + CODEX_API_KEY: "synthetic-live-key" })); + assert.equal(restoredLegacy.codexOptions.env.CODEX_HOME, "/fixture/runtime-home"); + assert.equal(restoredLegacy.codexOptions.env.CODEX_CLI_PATH, "/fixture/runtime-codex"); + assert.equal(restoredLegacy.codexOptions.config.model_reasoning_summary, "concise"); + await assert.rejects(readFile(join(legacyDir, "artifacts/deep_discovery/execution-settings.json")), + { code: "ENOENT" }); + } + await assert.rejects(loadSettings(join(root, "missing-v2"), { workflowVersion: "deep-security-scan/v2" }, + async () => assert.fail("missing promised v2 settings must not become legacy recovery")), /no recorded original/); + const [rebound, unboundLegacy] = await Promise.all([ + captureSettings({ usageOwner: originalOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }), + captureSettings({ model: "stored-model", usageOwner: null }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:03:00Z" }) + ]); + assert.equal(rebound.modelProvider, "openai", "takeover uses the recorded owner, not the invoking conversation"); + assert.equal(rebound.model, "parent-model", "the bound turn takes precedence over later turns"); + assert.equal(rebound.reasoningSummary, "none"); + assert.equal(unboundLegacy.model, "stored-model"); + assert.equal(unboundLegacy.modelProvider, undefined, "unrecorded legacy ownership cannot recover caller selections"); + assert.equal(unboundLegacy.reasoningSummary, undefined); + await writeFile(join(sessionDirectory, "legacy-auto.jsonl"), [ + { type: "session_meta", payload: { id: "fixture-legacy-auto", cli_version: "0.132.0", model_provider: "openai" } }, + { type: "turn_context", payload: { turn_id: "legacy-turn", model: "legacy-model", effort: "high", summary: "auto" } } + ].map(JSON.stringify).join("\n") + "\n"); + const legacyAuto = await captureSettings({ usageOwner: { threadId: "fixture-legacy-auto", turnId: "legacy-turn" } }, + { filesystemDenies: [] }, parentEnvironment); + assert.equal(legacyAuto.reasoningSummary, "auto", "older native turn-context selections remain readable"); + for (const version of ["0.133.0", "0.154.0"]) { + const threadId = `fixture-fresh-${version}`; + await writeFile(join(sessionDirectory, `${threadId}.jsonl`), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", payload: { id: threadId, cli_version: version, model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:01Z", payload: { turn_id: "fresh-turn", model: "fresh-model", effort: "high", summary: "auto" } }, + { type: "event_msg", timestamp: "2026-01-01T00:02:00Z", payload: { type: "thread_settings_applied", thread_id: threadId, + thread_settings: { model: "fresh-model", model_provider_id: "openai", reasoning_summary: "detailed" } } } + ].map(JSON.stringify).join("\n") + "\n"); + const owner = { threadId, turnId: "fresh-turn", startedAt: "2026-01-01T00:01:00Z" }; + const fresh = await captureSettings({ usageOwner: owner }, { filesystemDenies: [] }, parentEnvironment); + assert.equal(fresh.model, "fresh-model"); + assert.equal(fresh.reasoningSummary, undefined, "fresh native compatibility auto is not an original selection"); + assert.equal(restoreSettings(fresh, { filesystemDenies: [] }).codexOptions.config.model_reasoning_summary, undefined); + const freshDir = join(root, threadId); + await writeSnapshot(freshDir, fresh); + const freshPath = join(freshDir, "artifacts", "deep_discovery", "execution-settings.json"); + const freshBytes = await readFile(freshPath, "utf8"); + assert.deepEqual(await loadSettings(freshDir, { usageOwner: owner, createdAt: owner.startedAt }), fresh); + assert.equal(await readFile(freshPath, "utf8"), freshBytes, "unknown summary is not replaced by a compatibility field or a later selection"); + await writeFile(join(root, "config.toml"), 'model_reasoning_summary = "auto"\n'); + const explicit = await captureSettings({ usageOwner: owner }, { filesystemDenies: [] }, parentEnvironment); + assert.equal(explicit.reasoningSummary, "auto", "an explicit original config selection still takes precedence"); + await writeFile(join(root, "config.toml"), ""); + } + await writeFile(join(sessionDirectory, "applied.jsonl"), [ + { type: "session_meta", timestamp: "2026-01-01T00:00:00Z", payload: { id: "fixture-applied", model_provider: "previous-provider" } }, + { type: "event_msg", timestamp: "2026-01-01T00:00:01Z", payload: { type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "applied-model", model_provider_id: "openai", service_tier: "default", reasoning_effort: "high", reasoning_summary: "concise" } } }, + { type: "turn_context", timestamp: "2026-01-01T00:00:02Z", payload: { turn_id: "applied-turn", model: "previous-model", effort: "low", summary: "none" } }, + { type: "event_msg", timestamp: "2026-01-01T00:00:03Z", payload: { type: "thread_settings_applied", thread_id: "fixture-copied-owner", + thread_settings: { model: "copied-model", model_provider_id: "copied-provider", service_tier: "flex", reasoning_summary: "detailed" } } }, + { type: "event_msg", timestamp: "2026-01-01T00:02:00Z", payload: { type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "later-model", model_provider_id: "later-provider", service_tier: "fast", reasoning_summary: "detailed" } } } + ].map(JSON.stringify).join("\n") + "\n"); + const appliedOwner = { threadId: "fixture-applied", turnId: "applied-turn", startedAt: "2026-01-01T00:00:00Z" }; + const applied = await captureSettings({ usageOwner: appliedOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(applied.serviceTier, "default", "original explicit standard routing survives later and copied snapshots"); + assert.equal(applied.reasoningSummary, "concise", "native applied summary overrides the legacy compatibility field"); + assert.equal(applied.modelProvider, "openai", "complete native snapshot replaces the session metadata provider"); + assert.equal(applied.model, "applied-model", "complete native snapshot replaces compatibility turn settings"); + assert.equal(applied.reasoningEffort, "high"); + assert.equal(applied.nativeServiceTierAbsent, undefined, "explicit native standard remains an explicit selection"); + const tierDir = join(root, "missing-tier"); + const { serviceTier: omittedTier, ...withoutTier } = applied; + assert.equal(omittedTier, "default"); + await writeSnapshot(tierDir, withoutTier); + const repairedTier = await loadSettings(tierDir, { + usageOwner: appliedOwner, createdAt: "2026-01-01T00:01:00Z" + }); + assert.equal(repairedTier.serviceTier, "default"); + await writeFile(join(sessionDirectory, "applied.jsonl"), (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ type: "event_msg", timestamp: "2026-01-01T00:00:04Z", payload: { + type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "applied-model", model_provider_id: "openai", reasoning_effort: "high", service_tier: "priority" } + } }) + "\n"); + const nativeTier = await captureSettings({ usageOwner: appliedOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(nativeTier.serviceTier, "priority", "an effective tier selected by native remains unchanged"); + assert.equal(nativeTier.nativeServiceTierAbsent, undefined); + await writeFile(join(sessionDirectory, "applied.jsonl"), (await readFile(join(sessionDirectory, "applied.jsonl"), "utf8")) + + JSON.stringify({ type: "event_msg", timestamp: "2026-01-01T00:00:05Z", payload: { + type: "thread_settings_applied", thread_id: "fixture-applied", + thread_settings: { model: "applied-model", model_provider_id: "openai", reasoning_effort: "high" } + } }) + "\n"); + const nativeDefaults = await captureSettings({ usageOwner: appliedOwner }, { filesystemDenies: [] }, parentEnvironment, + { threadId: "fixture-other", startedAt: "2026-01-01T00:01:00Z" }); + assert.equal(nativeDefaults.model, "applied-model", "absent optional selections do not erase the required model"); + assert.equal(nativeDefaults.modelProvider, "openai", "absent optional selections do not erase the required provider"); + assert.equal(nativeDefaults.reasoningEffort, "high"); + assert.equal(nativeDefaults.serviceTier, "default", "known native absence retains its omitted request tier"); + assert.equal(nativeDefaults.nativeServiceTierAbsent, true, "known native absence is recorded separately from explicit standard"); + assert.equal(nativeDefaults.reasoningSummary, undefined, "a compatibility summary is not a recorded native default"); + const incompleteDir = join(root, "incomplete"); + const incomplete = { codexPath: process.execPath, codexHome: root, serviceTier: "flex" }; + await writeSnapshot(incompleteDir, incomplete); + await writeFile(join(root, "config.toml"), 'model_provider = "observer-provider"\nmodel_reasoning_summary = "detailed"\n'); + const originalRun = { model: "stored-model", reasoningEffort: "ultra", usageOwner: originalOwner, + createdAt: "2026-01-01T00:01:00Z" }; + const raceDir = join(root, "concurrent-recovery"); + await writeSnapshot(raceDir, incomplete); + const racePath = join(raceDir, "artifacts", "deep_discovery", "execution-settings.json"); + const originalCreateReadStream = fs.createReadStream; + const readingHistory = Promise.withResolvers(); + const releaseHistory = Promise.withResolvers(); + let held = false; + let pendingRead; + fs.createReadStream = (path, options) => { + const source = originalCreateReadStream(path, options); + if (path !== join(sessionDirectory, "parent.jsonl") || held) return source; + held = true; + const delayed = new PassThrough(); + source.once("error", (error) => delayed.destroy(error)); + delayed.once("close", () => source.destroy()); + void releaseHistory.promise.then(() => source.pipe(delayed)); + readingHistory.resolve(); + return delayed; + }; + syncBuiltinESMExports(); + try { + pendingRead = loadSettings(raceDir, originalRun); + await Promise.race([readingHistory.promise, pendingRead.then(() => + assert.fail("historical recovery must reach the controlled history read"))]); + const newer = { ...settings, modelProvider: "newer-provider", reasoningSummary: "concise" }; + await writeSnapshot(raceDir, newer); + const newerBytes = await readFile(racePath, "utf8"); + releaseHistory.resolve(); + const delayedProjection = await pendingRead; + assert.equal(delayedProjection.modelProvider, "openai"); + assert.equal(delayedProjection.reasoningSummary, "none"); + assert.equal(await readFile(racePath, "utf8"), newerBytes, + "a delayed historical projection must not overwrite a newer snapshot"); + assert.deepEqual(await loadSettings(raceDir), newer); + } finally { + releaseHistory.resolve(); + await pendingRead?.catch(() => {}); + fs.createReadStream = originalCreateReadStream; + syncBuiltinESMExports(); + } + const incompletePath = join(incompleteDir, "artifacts", "deep_discovery", "execution-settings.json"); + const incompleteBytes = await readFile(incompletePath, "utf8"); + const repaired = await loadSettings(incompleteDir, originalRun); + assert.deepEqual(repaired, { ...incomplete, model: "stored-model", reasoningEffort: "ultra", + modelProvider: "openai", reasoningSummary: "none" }); + assert.equal(await readFile(incompletePath, "utf8"), incompleteBytes, + "recovering historical fields is read-only"); + await rm(sessionDirectory, { recursive: true }); + const unavailable = await loadSettings(incompleteDir, originalRun); + assert.equal(unavailable.model, "stored-model"); + assert.equal(unavailable.reasoningEffort, "ultra"); + assert.equal(unavailable.modelProvider, undefined, "unavailable history remains unknown"); + assert.equal(unavailable.reasoningSummary, undefined); + assert.equal(await readFile(incompletePath, "utf8"), incompleteBytes); + const unknownDir = join(root, "unknown"); + await writeSnapshot(unknownDir, incomplete); + const unknown = await loadSettings(unknownDir, { ...originalRun, usageOwner: null }); + assert.equal(unknown.model, "stored-model"); + assert.equal(unknown.modelProvider, undefined, "missing original ownership is not current config"); + assert.equal(unknown.reasoningSummary, undefined); + assert.equal(unknown.nativeServiceTierAbsent, undefined); + const unsupported = JSON.stringify({ version: 99, settings }); + await writeFile(savedPath, unsupported); + await assert.rejects(loadSettings(join(root, "one")), /unsupported/); + assert.equal(await readFile(savedPath, "utf8"), unsupported); +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs new file mode 100644 index 000000000..d805d6f74 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_coverage.mjs @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { materialRemediations, materialRemediationTests, publishCoverageFixture } from "./deep_scan_coverage_fixture.mjs"; + +for (const splitSeededReducers of [false, true]) { + test(`selected publication retains fixes and coverage after failure and a rejected checkpoint (reducer chain: ${splitSeededReducers})`, async () => { + const root = await mkdtemp(path.join(tmpdir(), "selected-coverage-publication-")); + try { + const fixture = path.join(root, "fixture"); + await mkdir(fixture, { mode: 0o700 }); + const { scanDir, terminal } = await publishCoverageFixture(fixture, "partial", { + resume: true, immutableInputs: true, materialFindings: true, discardMutableResults: true, + splitSeededReducers, selectedRecovery: true, + }); + assert.equal(terminal.workflowVersion, "deep-security-scan/v2"); + assert.match(terminal.finalizationInput.resultPath, /checkpoints/); + const report = await readFile(path.join(scanDir, "report.md"), "utf8"); + for (const fix of [...materialRemediations, ...materialRemediationTests]) { + assert.equal(report.split(fix).length - 1, 1); + } + const coverage = JSON.parse(await readFile(path.join(scanDir, "coverage.json"), "utf8")); + assert.equal(coverage.completeness, "partial"); + assert.deepEqual(coverage.reviews.map((review) => review.completeness), ["partial", "complete", "unknown"]); + assert.equal(new Set(coverage.deferred.map((item) => item.candidateId)).size, 2); + for (const item of coverage.deferred) assert.ok(report.includes(item.reason)); + for (const surface of coverage.surfaces) { + assert.equal(await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"), "Synthetic review evidence.\n"); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs new file mode 100644 index 000000000..f982f021f --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selected_replay.mjs @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [path.resolve(import.meta.dirname, "../src/deep-scan/coordinator.ts")], + loader: { ".md": "text" }, + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=deep-scan-selected-replay.js" }, +}); +const { DeepScanCoordinator } = await import( + `data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`, +); + +for (const terminalReason of ["saturated", "capped"]) { + test(`restart publishes the selected ${terminalReason} result after an output failure`, async () => { + const scanDir = await realpath(await mkdtemp(path.join(tmpdir(), "selected-replay-"))); + try { + const scanId = "4e7b4acb-ac80-4d68-98cd-3d5ac5581cd1"; + const draft = { + scanId, complete: true, findings: [], + coverage: { completeness: "partial", surfaces: [], explicitExclusions: [], + deferred: [{ id: "review", reason: "A dependency remains unreviewed." }] }, + }; + const { coverage, ...reduction } = draft; + const bytes = JSON.stringify({ ...reduction, sourceCoverage: coverage }); + const digest = createHash("sha256").update(bytes).digest("hex"); + const resultPath = `artifacts/deep_discovery/dedup/dedup-0001/output/checkpoints/${digest}.json`; + await mkdir(path.dirname(path.join(scanDir, resultPath)), { recursive: true }); + await writeFile(path.join(scanDir, resultPath), bytes); + const selection = { version: 1, resultPath, resultSha256: digest, terminalReason, + omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z" }; + let run = { + scanId, scanDir, targetPath: scanDir, scope: ".", workflowVersion: "deep-security-scan/v2", + status: "running", phase: "terminal", coordinatorGeneration: 3, + finalizationInput: selection, terminalReason, createdAt: "2026-01-01T00:00:00Z", + config: { workers: 2, subagents: 0, stopAfterNoNew: 2, stopAfterConsecutiveErrors: 2, + maxDiscoveryRuns: 4, maxTimeHours: 1 }, + dispatchedCount: 4, noNewStreak: 2, consecutiveErrors: 0, + }; + const mutations = []; + const store = new Proxy({ + get: async () => structuredClone(run), + finish: async (input) => { + mutations.push("finish"); + assert.equal(input.reason, terminalReason); + assert.deepEqual(input.omittedWorkerIds, selection.omittedWorkerIds); + run = { ...run, status: "succeeded", manifestPath: input.manifestPath }; + return structuredClone(run); + }, + fail: async () => { mutations.push("fail"); run = { ...run, status: "failed" }; return run; }, + }, { get: (target, key) => key in target ? target[key] : async () => { + mutations.push(key); throw new Error(`Unexpected scheduler operation: ${String(key)}`); + } }); + let executions = 0; + let publications = 0; + const options = { + store, executor: { run: async () => { executions++; throw new Error("Unexpected model work"); } }, + pluginRoot: scanDir, threadId: "original-result-conversation", retryDelaysMs: [], + // Already expired: replay must not start a discovery deadline timer. + discoveryTimeoutMs: 1, + onComplete: async (actual, _signal, publication) => { + publications++; + assert.deepEqual(actual, draft); + assert.equal(publication.coordinatorGeneration, 3); + assert.equal(publication.resultPath, path.join(scanDir, resultPath)); + if (publications === 1) throw new Error("Synthetic publication write failure"); + }, + }; + const first = new DeepScanCoordinator({ ...options, run: structuredClone(run) }); + first.start(); + await assert.rejects(first.wait(), /Synthetic publication write failure/); + assert.equal(run.status, "running"); + assert.equal(run.terminalReason, terminalReason); + assert.deepEqual(run.finalizationInput, selection); + const restarted = new DeepScanCoordinator({ ...options, run: structuredClone(run) }); + restarted.start(); + const completed = await restarted.wait(); + assert.equal(completed.status, "succeeded"); + assert.equal(completed.terminalReason, terminalReason); + assert.equal(executions, 0); + assert.deepEqual(mutations, ["finish"]); + assert.equal(publications, 2); + } finally { + await rm(scanDir, { recursive: true, force: true }); + } + }); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs new file mode 100644 index 000000000..4f9300233 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_selection_store.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import { test } from "node:test"; +import { build } from "esbuild"; + +const bundle = await build({ + entryPoints: [path.resolve(import.meta.dirname, "../src/deep-scan/store.ts")], + bundle: true, format: "esm", platform: "node", write: false, + footer: { js: "//# sourceURL=deep-scan-selection-store.js" }, +}); +const { WorkbenchDeepScanStore } = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString("base64")}`); + +test("selection uses the dedicated function bridge and the store's existing replay policy", async () => { + const scanId = "ed8ff2da-01d9-4338-aeba-8bcfd4b530a9"; + const selection = { + version: 1, resultPath: "artifacts/merge/checkpoints/aggregate.json", resultSha256: "a".repeat(64), + terminalReason: "saturated", omittedWorkerIds: [], selectedAt: "2026-01-01T00:00:00Z", + }; + const response = { deepScan: { + scanId, targetPath: "/target", scope: ".", scanDir: "/scan", status: "running", + schemaVersion: 1, workflowVersion: "deep-security-scan/v2", coordinatorGeneration: 2, + config: { workers: 2, subagents: 0, stopAfterNoNew: 2, stopAfterConsecutiveErrors: 2, maxDiscoveryRuns: 4 }, + dispatchedCount: 2, noNewStreak: 2, consecutiveErrors: 0, finalizationInput: selection, + } }; + const calls = []; + const store = new WorkbenchDeepScanStore(async (...args) => { + calls.push(structuredClone(args)); + if (calls.length === 1) throw Object.assign(new Error("Synthetic lost selection response"), { code: "ETIMEDOUT" }); + return response; + }); + const result = await store.selectFinalization({ + scanId, coordinatorGeneration: 2, reason: "saturated", manifestPath: "/scan/scan-manifest.json", + resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json", omittedWorkerIds: [], + }); + assert.deepEqual(result.finalizationInput, selection); + assert.equal(calls.length, 2); + assert.deepEqual(calls[1], calls[0]); + assert.deepEqual(calls[0], [[ + "finish-deep-scan", "--scan-id", scanId, "--coordinator-generation", "2", + "--terminal-reason", "saturated", "--manifest-path", "/scan/scan-manifest.json", + ], JSON.stringify({ resultPath: "/scan/artifacts/merge/checkpoints/aggregate.json" }), true]); +}); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs index e2a0fd3cd..69880a523 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFile, spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { chmod, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -10,7 +10,10 @@ import { build } from "esbuild"; const execFileAsync = promisify(execFile); const mcpAppRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const pluginRoot = path.resolve(mcpAppRoot, ".."); +const installedPluginRoot = process.env.CODEX_SECURITY_TEST_PLUGIN_ROOT; +const pluginRoot = installedPluginRoot + ? path.resolve(installedPluginRoot) + : path.resolve(mcpAppRoot, ".."); const workbenchPath = path.join(pluginRoot, "scripts", "workbench_db.py"); const parentSandboxState = { permissionProfile: { @@ -30,9 +33,284 @@ const parentSandboxState = { if (process.platform === "win32") { console.log("deep scan stdio lifecycle test skipped on Windows (POSIX fake Codex executable)"); } else { + for (const mode of ["cancel-after-seal", "cancel-finalizer", "cancel-finalizer-failure", "late-rejoin", "late-rejoin-failure", "detached", "joined", "remote", "failure", "active-failure", "remote-replay", "lost-response", "lost-response-corrupt", "lost-response-remove"]) { + await testDeepScanDetachedCompletion(mode); + } await testDeepScanStdioLifecycle(); } +async function testDeepScanDetachedCompletion(mode) { + const fixtureRoot = await mkdtemp(path.join(tmpdir(), "codex-security-deep-detached-")); + const targetPath = path.join(fixtureRoot, "target"); + const stateDir = path.join(fixtureRoot, "state"); + const scanRoot = path.join(fixtureRoot, "scans"); + const codexHome = path.join(fixtureRoot, "codex-home"); + const startLogPath = path.join(fixtureRoot, "started.jsonl"); + const exitLogPath = path.join(fixtureRoot, "exited.jsonl"); + const controlPath = path.join(fixtureRoot, "completion-control"); + const finalizerControlPath = path.join(fixtureRoot, "finalizer-control"); + const finalizerLogPath = path.join(fixtureRoot, "finalizer.jsonl"); + const committedControlPath = path.join(fixtureRoot, "committed-control"); + const committedLogPath = path.join(fixtureRoot, "committed.jsonl"); + const pythonWrapperPath = path.join(fixtureRoot, "python-wrapper.mjs"); + const fakeCodexPath = path.join(fixtureRoot, "fake-codex.mjs"); + const serverBundlePath = path.join(pluginRoot, "mcp", + installedPluginRoot ? "server.mjs" : `.deep-scan-detached-${randomUUID()}.cjs`); + const threadId = "deep-scan-detached-result-conversation"; + for (const directory of [targetPath, stateDir, scanRoot, path.join(codexHome, "codex-security")]) { + await mkdir(directory, { recursive: true }); + } + await writeFile(path.join(targetPath, "fixture.py"), "print('fixture')\n"); + await writeFile(path.join(codexHome, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = 1\nmax_discovery_runs = 2\n"); + await writeFile(controlPath, "wait-for-completion"); + await writePythonWrapper(pythonWrapperPath); + if (mode === "cancel-after-seal") await writeFile(committedControlPath, "wait"); + if (mode !== "detached") await writeFile(finalizerControlPath, mode === "joined" || mode === "remote" || mode.startsWith("cancel-finalizer") || mode.startsWith("late-rejoin") ? "wait" : mode === "active-failure" || mode === "remote-replay" ? "failure" : mode.startsWith("lost-response") ? "lost-response" : mode); + await writeFakeCodex(fakeCodexPath); + if (!installedPluginRoot) await bundleServer(serverBundlePath); + const environment = { + ...process.env, + OPENAI_API_KEY: "synthetic-stdio-key", CODEX_API_KEY: "", + CODEX_CLI_PATH: fakeCodexPath, CODEX_HOME: codexHome, + CODEX_SECURITY_SCAN_ROOT: scanRoot, CODEX_SECURITY_STATE_DIR: stateDir, + FAKE_CODEX_START_LOG: startLogPath, FAKE_CODEX_EXIT_LOG: exitLogPath, + FAKE_CODEX_RESTART_CONTROL: controlPath, + PYTHON: pythonWrapperPath, REAL_PYTHON: process.env.PYTHON?.trim() || "python3", + FAKE_WORKBENCH_FINALIZER_CONTROL: finalizerControlPath, + FAKE_WORKBENCH_FINALIZER_LOG: finalizerLogPath, + FAKE_WORKBENCH_CANCEL_LOG: path.join(fixtureRoot, "cancel.jsonl"), + FAKE_WORKBENCH_COMMITTED_CONTROL: committedControlPath, + FAKE_WORKBENCH_COMMITTED_LOG: committedLogPath, + FAKE_CODEX_SIGNAL_CHECKPOINT_CONTROL: path.join(fixtureRoot, "unused-signal-control"), + }; + const server = startServer(serverBundlePath, environment); + let remote; + try { + assertNoError(await server.request(1, "initialize", { + protocolVersion: "2025-11-25", capabilities: {}, + clientInfo: { name: "deep-scan-detached-completion", version: "0.1.0" }, + })); + server.sendRequest(2, "tools/call", toolCall("start_codex_security_deep_scan", + { targetPath, scope: ".", userContext: "Original discovery input" }, threadId)); + const scanId = await waitForScanId({ server, requestId: 2 }); + await waitForDeepScanWorker({ environment, scanId, threadId }); + const [worker] = await waitForJsonLines(startLogPath, 1); + if (mode === "joined") { + server.sendRequest(3, "tools/call", toolCall("start_codex_security_deep_scan", { scanId }, threadId)); + await waitFor(() => server.stderrEvents().some((event) => event.event === "coordinator_joined"), + "second observer to join"); + } + if (mode === "remote") { + remote = startServer(serverBundlePath, environment); + assertNoError(await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", capabilities: {}, + clientInfo: { name: "remote-observer", version: "0.1.0" }, + })); + remote.sendRequest(2, "tools/call", toolCall("start_codex_security_deep_scan", { scanId }, threadId)); + await waitFor(() => remote.stderrEvents().some((event) => event.event === "coordinator_joined"), + "remote request to observe the existing owner"); + } + if (mode !== "active-failure" && mode !== "cancel-after-seal" && !mode.startsWith("cancel-finalizer") && !mode.startsWith("late-rejoin")) server.notify("notifications/cancelled", { requestId: 2, reason: "detach original observer" }); + const detached = await getDeepScan({ environment, scanId, threadId }); + const originalPublicScan = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + assert.equal(detached.status, "running"); + assert.equal(detached.cancelRequested, false); + assertProcessAlive(worker.pid); + assertProcessAlive(server.pid); + + // The already-owned workers proceed after observation ends. + await writeFile(controlPath, "after-restart"); + let finished; + await waitFor(async () => { + finished = await getDeepScan({ environment, scanId, threadId }); + return finished.status === "succeeded"; + }, "detached discovery and reducer to finish"); + assert.equal(finished.workflowVersion, "deep-security-scan/v2"); + assert.equal(finished.finalizationInput.version, 1); + assert.equal(finished.coordinatorGeneration, detached.coordinatorGeneration); + assert.equal(finished.userContext, detached.userContext); + assert.deepEqual(finished.usageOwner, detached.usageOwner); + assert.equal(finished.workers.filter((row) => row.kind === "discovery" && row.status === "succeeded").length, 2); + assert.equal(finished.workers.filter((row) => row.kind === "dedup" && row.status === "succeeded").length, 1); + const selectedBytes = await readFile(path.join(finished.scanDir, finished.finalizationInput.resultPath)); + assert.equal(createHash("sha256").update(selectedBytes).digest("hex"), finished.finalizationInput.resultSha256); + if (mode === "cancel-after-seal") { + await waitForJsonLines(committedLogPath, 1); + const beforeCancel = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + assert.equal(beforeCancel.scan.progress.status, "complete"); + const sealedBytes = await readFile(path.join(finished.scanDir, "scan-manifest.json")); + server.sendRequest(6, "tools/call", toolCall("cancel_codex_security_scan", { scanId }, threadId)); + await waitFor(() => server.stderrEvents().some((event) => event.event === "coordinator_cancel_requested"), + "cancellation after parent completion committed"); + await rm(committedControlPath, { force: true }); + const cancellation = await server.waitForResponse(6); + assert.equal(cancellation.result?.isError, true); + const afterCancel = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + assert.equal(afterCancel.scan.progress.status, "complete"); + assert.equal(afterCancel.scan.executionAttribution.completedAt, beforeCancel.scan.executionAttribution.completedAt); + assert.deepEqual(await readFile(path.join(finished.scanDir, "scan-manifest.json")), sealedBytes); + return; + } + if (mode.startsWith("cancel-finalizer")) { + await waitForJsonLines(finalizerLogPath, 1); + server.sendRequest(6, "tools/call", toolCall("cancel_codex_security_scan", { scanId }, threadId)); + await waitFor(() => server.stderrEvents().some((event) => event.event === "coordinator_cancel_requested"), + "owner cancellation before releasing parent completion"); + if (mode === "cancel-finalizer-failure") await writeFile(finalizerControlPath, "failure"); + else await rm(finalizerControlPath, { force: true }); + const canceled = await server.waitForResponse(6); + const ownerResponse = await server.waitForResponse(2); + const publicScan = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + const stopped = await getDeepScan({ environment, scanId, threadId }); + assert.equal(publicScan.scan.progress.status, "canceled", "cancellation observed before parent completion must remain canceled"); + assertNoError(canceled); + assertCanceled(ownerResponse, scanId); + assert.equal(stopped.status, "canceled"); + assert.deepEqual(await readFile(path.join(finished.scanDir, finished.finalizationInput.resultPath)), selectedBytes); + assert.deepEqual(stopped.finalizationInput, finished.finalizationInput); + assert.equal(publicScan.scan.continuationThreadId, originalPublicScan.scan.continuationThreadId); + return; + } + if (mode.startsWith("late-rejoin")) { + await waitForJsonLines(finalizerLogPath, 1); + remote = startServer(serverBundlePath, { + ...environment, + FAKE_WORKBENCH_FINALIZER_CONTROL: path.join(fixtureRoot, "remote-finalizer-control"), + }); + assertNoError(await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", capabilities: {}, + clientInfo: { name: "late-remote-observer", version: "0.1.0" }, + })); + remote.sendRequest(2, "tools/call", toolCall("start_codex_security_deep_scan", { scanId }, threadId)); + const observed = await remote.waitForResponse(2); + const during = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + if (mode === "late-rejoin-failure") await writeFile(finalizerControlPath, "failure"); + else await rm(finalizerControlPath); + const ownerResponse = await server.waitForResponse(2); + const invocations = await readJsonLines(finalizerLogPath); + assertNoError(observed); + assert.equal(observed.result.structuredContent.manifestPath, path.join(finished.scanDir, "scan-manifest.json")); + assert.deepEqual(await readFile(path.join(finished.scanDir, finished.finalizationInput.resultPath)), selectedBytes); + assert.equal(invocations.length, 1, "late remote observation must not invoke a competing parent finalizer"); + assert.equal(during.scan.progress.status, "running", "only the existing owner may finish pending parent completion"); + if (mode === "late-rejoin-failure") { + assert.equal(ownerResponse.result?.isError, true); + assert.equal((await runWorkbench(environment, ["get-scan", "--scan-id", scanId])).scan.progress.status, "running"); + return; + } + assertNoError(ownerResponse); + } + if (mode === "joined") { + await waitForJsonLines(finalizerLogPath, 1); + // Rejoin while finish-deep-scan is committed but public sealing is blocked. + server.sendRequest(4, "tools/call", toolCall("start_codex_security_deep_scan", { scanId }, threadId)); + await waitFor(() => server.stderrEvents().filter((event) => event.event === "coordinator_joined").length === 2, + "observer to join pending public completion"); + await delay(5_100); // Exercise an actual coordinator heartbeat during finalization. + assert.equal(server.response(3), undefined); + assert.equal(server.response(4), undefined); + assert.equal((await runWorkbench(environment, ["get-scan", "--scan-id", scanId])).scan.progress.status, "running"); + assert.equal((await readJsonLines(finalizerLogPath)).length, 1); + await rm(finalizerControlPath); + for (const id of [3, 4]) assertNoError(await server.waitForResponse(id)); + } + if (mode === "remote") { + // A remote observer retains the aggregate-ready response. It cannot run + // the owning process's public finalizer, even after Deep itself succeeds. + const observed = await remote.waitForResponse(2); + await waitForJsonLines(finalizerLogPath, 1); + assertNoError(observed); + assert.equal(observed.result.structuredContent.manifestPath, path.join(finished.scanDir, "scan-manifest.json")); + assert.equal((await readJsonLines(finalizerLogPath)).length, 1); + await rm(finalizerControlPath); + } + if (mode === "failure" || mode === "active-failure" || mode === "remote-replay" || mode.startsWith("lost-response")) { + await waitFor(() => server.stderrEvents().some((event) => event.event === "coordinator_publication_pending"), + "public finalization failure to remain pending"); + if (mode === "active-failure") { + const failed = await server.waitForResponse(2); + assert.equal(failed.result?.isError, true); + const message = failed.result.content.map((item) => item.text).join(" "); + assert.match(message, /injected complete-scan failure/); + assert.match(message, /Do not call start_codex_security_deep_scan again in this response/); + assert.match(message, /no final|Do not.*final/i); + } + const pending = await getDeepScan({ environment, scanId, threadId }); + assert.equal(pending.status, "succeeded"); + assert.equal(pending.terminalReason, finished.terminalReason); + assert.deepEqual(pending.finalizationInput, finished.finalizationInput); + assert.equal((await readJsonLines(finalizerLogPath)).length, 1, "the owner does not add a retry layer"); + const beforeReplay = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + assert.equal(beforeReplay.scan.progress.status, !mode.startsWith("lost-response") ? "running" : "complete"); + const manifestBeforeReplay = await readFile(path.join(finished.scanDir, "scan-manifest.json")); + let replayServer = server; + if (mode === "remote-replay") { + await server.stop(); + remote = startServer(serverBundlePath, environment); + replayServer = remote; + assertNoError(await remote.request(1, "initialize", { + protocolVersion: "2025-11-25", capabilities: {}, + clientInfo: { name: "reconstructed-finalizer", version: "0.1.0" }, + })); + } + const damagedArtifact = path.join(finished.scanDir, "findings.json"); + if (mode === "lost-response-corrupt") { + await writeFile(damagedArtifact, Buffer.concat([await readFile(damagedArtifact), Buffer.from(" ")])); + } else if (mode === "lost-response-remove") { + await rm(damagedArtifact); + } + const rejoined = await replayServer.request(5, "tools/call", toolCall("start_codex_security_deep_scan", { scanId }, threadId)); + if (mode === "lost-response-corrupt" || mode === "lost-response-remove") { + assert.equal(rejoined.result?.isError, true, "completion replay must reject altered sealed artifacts"); + assert.match(rejoined.result.content.map((item) => item.text).join(" "), + /sealed artifact changed or is missing|findings\.json: expected a file/); + assert.deepEqual(await readFile(path.join(finished.scanDir, "scan-manifest.json")), manifestBeforeReplay); + assert.deepEqual(await readFile(path.join(finished.scanDir, finished.finalizationInput.resultPath)), selectedBytes); + assert.equal((await readJsonLines(startLogPath)).length, 3, "rejected replay launches no workers"); + const afterReplay = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + assert.equal(afterReplay.scan.progress.status, "complete"); + assert.equal(afterReplay.scan.executionAttribution.completedAt, beforeReplay.scan.executionAttribution.completedAt); + console.log("native selected completion integrity passed", mode, scanId); + return; + } + assertNoError(rejoined); + assert.equal(rejoined.result.structuredContent.manifestPath, path.join(finished.scanDir, "scan-manifest.json")); + if (mode.startsWith("lost-response")) { + assert.deepEqual(await readFile(path.join(finished.scanDir, "scan-manifest.json")), manifestBeforeReplay); + const afterReplay = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + assert.equal(afterReplay.scan.progress.status, "complete"); + assert.equal(afterReplay.scan.executionAttribution.completedAt, beforeReplay.scan.executionAttribution.completedAt); + } + assert.deepEqual((await getDeepScan({ environment, scanId, threadId })).finalizationInput, finished.finalizationInput); + } + let publicScan; + await waitFor(async () => { + publicScan = await runWorkbench(environment, ["get-scan", "--scan-id", scanId]); + return publicScan.scan.progress.status === "complete"; + }, "public completion without another observer"); + assert.equal(publicScan.scan.progress.status, "complete"); + const manifest = JSON.parse(await readFile(path.join(finished.scanDir, "scan-manifest.json"), "utf8")); + assert.equal(typeof manifest.scan.sealedAt, "string"); + assert.equal(typeof manifest.scan.completedAt, "string"); + assert.equal(publicScan.scan.continuationThreadId, originalPublicScan.scan.continuationThreadId); + assert.deepEqual(publicScan.scan.executionAttribution.owner, originalPublicScan.scan.executionAttribution.owner); + assert.equal(publicScan.scan.executionAttribution.owner.threadId, threadId); + assert.equal((await readJsonLines(startLogPath)).length, 3, "completion and replay launch no extra model workers"); + assert.equal((await readJsonLines(finalizerLogPath)).length, mode === "failure" || mode === "active-failure" || mode === "remote-replay" ? 2 : 1); + assertProcessAlive(mode === "remote-replay" ? remote.pid : server.pid); + console.log("native selected completion passed", mode, scanId); + } catch (error) { + error.message += `\nMCP stderr:\n${server.stderrText()}`; + throw error; + } finally { + await remote?.stop(); + await server.stop(); + if (!installedPluginRoot) await rm(serverBundlePath, { force: true }); + await rm(fixtureRoot, { recursive: true, force: true }); + } +} + async function testDeepScanStdioLifecycle() { const fixtureRoot = await mkdtemp(path.join(tmpdir(), "codex-security-deep-stdio-")); const targetPath = path.join(fixtureRoot, "target"); @@ -55,7 +333,7 @@ async function testDeepScanStdioLifecycle() { const serverBundlePath = path.join( pluginRoot, "mcp", - `.deep-scan-stdio-test-${randomUUID()}.cjs` + installedPluginRoot ? "server.mjs" : `.deep-scan-stdio-test-${randomUUID()}.cjs` ); const threadId = "deep-scan-stdio-lifecycle-thread"; @@ -86,7 +364,7 @@ async function testDeepScanStdioLifecycle() { '' ].join('\n')); await writePythonWrapper(pythonWrapperPath); - await bundleServer(serverBundlePath); + if (!installedPluginRoot) await bundleServer(serverBundlePath); const environment = { ...process.env, @@ -429,7 +707,7 @@ async function testDeepScanStdioLifecycle() { "the MCP server must remain responsive after canceling one scan" ); - const resumedThreadId = "deep-scan-stdio-resumed-thread"; + let resumedThreadId = "deep-scan-stdio-resumed-thread"; const opened = await server.request(24, "tools/call", toolCall( "open_codex_security_workspace", { targetPath, scope: ".", mode: "deep" }, @@ -439,7 +717,7 @@ async function testDeepScanStdioLifecycle() { const sessionId = opened.result.structuredContent.workspace.id; assertNoError(await server.request(25, "tools/call", toolCall( "submit_codex_security_setup", - { sessionId, targetPath, scope: ".", mode: "deep" }, + { sessionId, targetPath, scope: ".", mode: "deep", userContext: "Original discovery focus" }, resumedThreadId ))); const started = await server.request(26, "tools/call", toolCall( @@ -450,7 +728,7 @@ async function testDeepScanStdioLifecycle() { assertNoError(started); const resumedScan = started.result.structuredContent.workspace.results; const resumedScanId = resumedScan.scanId; - const handoffClaimToken = randomUUID(); + let handoffClaimToken = randomUUID(); for (const [id, name, arguments_] of [ [27, "claim_codex_security_scan_handoff_delivery", { scanId: resumedScanId, claimToken: handoffClaimToken @@ -484,6 +762,16 @@ async function testDeepScanStdioLifecycle() { const completedDraft = JSON.parse(await readFile(completedWorker.resultManifestPath, "utf8")); assert.equal(completedDraft.scanId, resumedScanId); assert.deepEqual(completedDraft.findings, []); + assert.equal(partial.workflowVersion, "deep-security-scan/v2", "new scans use selected finalization by default"); + assert.equal(partial.userContext, "Original discovery focus"); + assert.equal(partial.usageOwner.threadId, resumedThreadId); + const settingsPath = path.join(resumedScan.scanDir, "artifacts", "deep_discovery", "execution-settings.json"); + const originalSettings = await readFile(settingsPath, "utf8"); + assertNoError(await server.request(30, "tools/call", toolCall( + "update_codex_security_scan_context", + { scanId: resumedScanId, handoffClaimToken, userContext: "Later result discussion" }, + resumedThreadId + ))); await server.stop(); assert.throws(() => process.kill(server.pid, 0), "the original MCP server must have exited"); const paused = await runWorkbench(environment, ["get-scan", "--scan-id", resumedScanId]); @@ -512,7 +800,21 @@ async function testDeepScanStdioLifecycle() { path.join(stateDir, "workbench.sqlite3"), resumedScanId ]); + await runWorkbench(environment, [ + "release-handoff-delivery", "--scan-id", resumedScanId, "--claim-token", handoffClaimToken + ]); + handoffClaimToken = randomUUID(); + resumedThreadId = "deep-scan-stdio-replacement-thread"; + await runWorkbench(environment, [ + "claim-handoff-delivery", "--scan-id", resumedScanId, "--claim-token", handoffClaimToken + ]); + await runWorkbench(environment, [ + "attach-scan-continuation-thread", "--scan-id", resumedScanId, + "--claim-token", handoffClaimToken, "--thread-id", resumedThreadId + ]); await writeFile(restartControlPath, "after-restart"); + // A replacement caller's configuration must not replace the original selection. + await writeFile(runtimeConfigPath, 'model_reasoning_summary = "detailed"\n'); const restartedServer = startServer(serverBundlePath, environment); try { @@ -537,8 +839,15 @@ async function testDeepScanStdioLifecycle() { environment, scanId: resumedScanId, threadId: resumedThreadId }); assert.equal(finished.status, "succeeded"); + assert.equal(finished.workflowVersion, partial.workflowVersion); + assert.equal(finished.finalizationInput.version, 1, "recovery selects a persisted finalization input"); assert.equal(finished.coordinatorGeneration, partial.coordinatorGeneration + 1); assert.equal(finished.dispatchedCount, 2); + assert.equal(finished.userContext, partial.userContext); + assert.equal(finished.createdAt, partial.createdAt, "recovery retains the original deadline origin"); + assert.equal(finished.config.maxTimeHours, partial.config.maxTimeHours); + assert.deepEqual(finished.usageOwner, partial.usageOwner, "a replacement continuation does not rebind original usage"); + assert.equal(await readFile(settingsPath, "utf8"), originalSettings); const successfulDiscoveries = finished.workers.filter((worker) => ( worker.kind === "discovery" && worker.status === "succeeded" )); @@ -557,6 +866,8 @@ async function testDeepScanStdioLifecycle() { const executions = (await readJsonLines(startLogPath)).slice(restartStartIndex); for (const execution of executions) { assert.equal(execution.argv.includes('model_reasoning_summary="none"'), true); + const context = discoveryPromptContext(execution.stdin); + if (context.workerLabel) assert.equal(context.userContext, "Original discovery focus"); } assert.equal(executions.filter((execution) => ( discoveryPromptContext(execution.stdin).workerLabel === "discovery-0001" @@ -569,7 +880,7 @@ async function testDeepScanStdioLifecycle() { throw error; } finally { await server.stop(); - await rm(serverBundlePath, { force: true }); + if (!installedPluginRoot) await rm(serverBundlePath, { force: true }); await rm(fixtureRoot, { recursive: true, force: true }); } } @@ -577,6 +888,7 @@ async function testDeepScanStdioLifecycle() { async function bundleServer(outfile) { await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { "import.meta.url": "__filename" }, entryPoints: [path.join(mcpAppRoot, "main.ts")], external: ["fsevents"], @@ -810,6 +1122,9 @@ async function writeFakeCodex(executablePath) { "const root = process.argv[process.argv.indexOf('--cd') + 1];", "appendFileSync(process.env.FAKE_CODEX_START_LOG, JSON.stringify({ pid: process.pid, argv: process.argv.slice(2), stdin, hasExpectedApiKey: process.env.CODEX_API_KEY === 'synthetic-stdio-key' }) + '\\n');", "console.log(JSON.stringify({ type: 'thread.started', thread_id: `stdio-fixture-${process.pid}` }));", + "while (existsSync(process.env.FAKE_CODEX_RESTART_CONTROL) && readFileSync(process.env.FAKE_CODEX_RESTART_CONTROL, 'utf8') === 'wait-for-completion') {", + " await new Promise((resolve) => setTimeout(resolve, 25));", + "}", "if (existsSync(process.env.FAKE_CODEX_RESTART_CONTROL)) {", " const phase = readFileSync(process.env.FAKE_CODEX_RESTART_CONTROL, 'utf8');", " if (phase === 'after-restart' || context.workerLabel === 'discovery-0001') {", @@ -849,7 +1164,7 @@ async function writeFakeCodex(executablePath) { async function writePythonWrapper(executablePath) { await writeFile(executablePath, [ "#!/usr/bin/env node", - 'import { appendFileSync, existsSync, unlinkSync } from "node:fs";', + 'import { appendFileSync, existsSync, readFileSync, unlinkSync } from "node:fs";', 'import { spawnSync } from "node:child_process";', "const args = process.argv.slice(2);", "const control = process.env.FAKE_WORKBENCH_CANCEL_FAILURE_CONTROL;", @@ -861,8 +1176,27 @@ async function writePythonWrapper(executablePath) { " console.error('injected cancel-scan failure');", " process.exit(1);", "}", + "let finalizerMode;", + "if (args[1] === 'complete-scan' && process.env.FAKE_WORKBENCH_FINALIZER_LOG) {", + " appendFileSync(process.env.FAKE_WORKBENCH_FINALIZER_LOG, JSON.stringify(args) + '\\n');", + " const finalizerControl = process.env.FAKE_WORKBENCH_FINALIZER_CONTROL;", + " while (existsSync(finalizerControl) && readFileSync(finalizerControl, 'utf8') === 'wait') {", + " await new Promise((resolve) => setTimeout(resolve, 25));", + " }", + " if (existsSync(finalizerControl)) {", + " finalizerMode = readFileSync(finalizerControl, 'utf8');", + " unlinkSync(finalizerControl);", + " }", + " if (finalizerMode === 'failure') { console.error('injected complete-scan failure'); process.exit(1); }", + "}", "const result = spawnSync(process.env.REAL_PYTHON || 'python3', args, { stdio: 'inherit' });", + "const committedControl = process.env.FAKE_WORKBENCH_COMMITTED_CONTROL;", + "if (args[1] === 'complete-scan' && result.status === 0 && committedControl && existsSync(committedControl)) {", + " appendFileSync(process.env.FAKE_WORKBENCH_COMMITTED_LOG, JSON.stringify({ pid: process.pid, committed: true }) + '\\n');", + " while (existsSync(committedControl)) await new Promise((resolve) => setTimeout(resolve, 25));", + "}", "if (result.error) throw result.error;", + "if (result.status === 0 && finalizerMode === 'lost-response') { console.error('injected lost completion response'); process.exit(1); }", "process.exit(result.status ?? 1);", "", ].join("\n")); diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs index 09eb1f884..8026cf600 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs @@ -17,10 +17,13 @@ const { WorkbenchDeepScanStore, parseDeepScan } = await import( ); await testBeginProtocolAndParsing(); +await testBeginCarriesOriginalSettingsWithUserContext(); +await testClaimChecksOriginalSettingsInsideTheTransaction(); await testCanonicalCommitProtocol(); await testTerminalProtocol(); testCanonicalNullAndPartialParsing(); testRunErrorParsing(); +testWorkflowVersionParsing(); testConfiguredMaximumDurationParsing(); await testWriteSerializationAndRecovery(); await testBeginUsesTheWriteQueue(); @@ -35,6 +38,46 @@ await testPersistenceRetryExhaustionPreservesDiagnostics(); await testDeterministicPersistenceFailuresAreNotRetried(); await testNonIdempotentMutationsAreNotRetried(); testInvalidPersistedConfig(); +testOriginalUsageOwnerParsing(); + +function testOriginalUsageOwnerParsing() { + const value = stateResult(randomUUID()); + const usageOwner = { threadId: "original-thread", turnId: "original-turn", startedAt: "2026-01-01T00:00:00Z" }; + assert.deepEqual(parseDeepScan({ deepScan: { ...value.deepScan, usageOwner } }).usageOwner, usageOwner); + assert.equal(parseDeepScan({ deepScan: { ...value.deepScan, usageOwner: null } }).usageOwner, null); + assert.equal(parseDeepScan(value).usageOwner, null, "old readers do not establish an original owner"); +} + +async function testBeginCarriesOriginalSettingsWithUserContext() { + const executionSettings = { codexPath: "/fixture/codex", codexHome: "/fixture/home", + model: "original-model", reasoningSummary: "concise" }; + const userContext = "Review the parser.\nKeep this second line."; + const runner = async (args, input, selectFinalization, withExecutionSettings) => { + assert.equal(selectFinalization, false); + assert.equal(withExecutionSettings, true); + assert.deepEqual(JSON.parse(input), { executionSettings, userContext }); + assert.equal(args.includes("--user-context-stdin"), false, + "the private structured input carries context without a second stdin consumer"); + assert.equal(args.some((arg) => arg.includes("execution-settings")), false, + "no public argument is added for the internal settings handoff"); + return stateResult(randomUUID(), { startDisposition: "created" }); + }; + await new WorkbenchDeepScanStore(runner).begin({ targetPath: "/fixture/repository", + threadId: "fixture-thread", scanRoot: "/fixture/scans", userContext, executionSettings }); +} + +async function testClaimChecksOriginalSettingsInsideTheTransaction() { + const scanId = randomUUID(); + const store = new WorkbenchDeepScanStore(async (args, input, selectFinalization, withExecutionSettings) => { + assert.equal(args[0], "claim-deep-scan-coordinator"); + assert.equal(input, undefined); + assert.equal(selectFinalization, false); + assert.equal(withExecutionSettings, true); + return { ...stateResult(scanId, { deepScan: { coordinatorGeneration: 2 } }), + coordinatorDisposition: "claimed" }; + }); + assert.equal((await store.claimCoordinator({ scanId, threadId: "fixture-thread" })).acquired, true); +} async function testBeginProtocolAndParsing() { const scanId = randomUUID(); @@ -71,7 +114,7 @@ async function testBeginProtocolAndParsing() { assert.equal(calls[0].input, "focus on archive parsing"); assert.equal(flagValue(calls[0].args, "--scan-root"), "/fixture/scans"); assert.equal(flagValue(calls[0].args, "--available-parallelism"), String(availableParallelism())); - assert.equal(flagValue(calls[0].args, "--workflow-version"), "deep-scan-mcp/v1"); + assert.equal(flagValue(calls[0].args, "--workflow-version"), "deep-security-scan/v2"); const claimToken = randomUUID(); let joinedArgs; @@ -424,7 +467,7 @@ async function testPersistenceRetriesRemainInsideTheWriteQueue() { if (args[0] === "claim-deep-scan-dedup" && calls.length === 1) { throw new Error("sqlite3.OperationalError: database is locked"); } - return {}; + return stateResult(scanId); }); const claim = store.claimDedup({ @@ -705,7 +748,7 @@ function idempotentPersistenceScenarios() { }) }, { operation: "claim-deep-scan-dedup", - result: {}, + result: stateResult(scanId), invoke: (store) => store.claimDedup({ id: reducerId, scanId, @@ -875,3 +918,31 @@ function deferred() { }); return { promise, resolve }; } + +function testWorkflowVersionParsing() { + const value = stateResult(randomUUID()).deepScan; + const run = parseDeepScan({ deepScan: { ...value, schemaVersion: 1, workflowVersion: "deep-scan-mcp/v1", model: "original-model", reasoningEffort: "high" } }); + assert.equal(run.model, "original-model"); + assert.equal(run.reasoningEffort, "high"); + assert.equal(run.schemaVersion, 1); + assert.equal(run.workflowVersion, "deep-scan-mcp/v1"); + const future = parseDeepScan({ deepScan: { ...value, schemaVersion: 99, workflowVersion: "future/v99" } }); + assert.equal(future.schemaVersion, 99); + assert.equal(future.workflowVersion, "future/v99", "inspection preserves unsupported versions"); +} + +for (const version of [1, 99]) { + const finalizationInput = { + version, + resultPath: null, + resultSha256: null, + terminalReason: "capped", + omittedWorkerIds: ["fixture-worker"], + selectedAt: "2026-01-01T00:00:00Z" + }; + assert.deepEqual( + parseDeepScan(stateResult(randomUUID(), { deepScan: { finalizationInput } })).finalizationInput, + finalizationInput, + "inspection preserves finalization input and version before execution compatibility checks" + ); +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs index a93384919..430e71600 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.mjs @@ -91,7 +91,10 @@ async function testRecoveredPublicationRejectsLateFailure() { paths: ["fixture.py"], }], }, - }, runWorkbench); + }, runWorkbench, undefined, { + coordinatorGeneration: claim.run.coordinatorGeneration, + resultPath: null, + }); await runWorkbench([ "cancel-scan", "--scan-id", run.scanId, "--thread-id", "publication-failure-owner", @@ -318,6 +321,7 @@ async function testReducerCommitAndFinishAgainstRealWorkbench() { CODEX_SECURITY_STATE_DIR: stateDir }; const python = process.env.PYTHON?.trim() || "python3"; + const finishCalls = []; const runWorkbench = async (args) => { const { stdout } = await execFileAsync(python, [workbenchPath, ...args], { cwd: pluginRoot, @@ -325,6 +329,12 @@ async function testReducerCommitAndFinishAgainstRealWorkbench() { maxBuffer: 4 * 1024 * 1024, timeout: 30_000 }); + if (args[0] === "finish-deep-scan") { + finishCalls.push([...args]); + if (finishCalls.length === 1) { + throw Object.assign(new Error("Synthetic lost committed finish response"), { code: "ETIMEDOUT" }); + } + } return JSON.parse(stdout); }; const store = new WorkbenchDeepScanStore(runWorkbench); @@ -505,6 +515,8 @@ async function testReducerCommitAndFinishAgainstRealWorkbench() { omittedWorkerIds: [late.id] }); assert.equal(finished.status, "succeeded"); + assert.equal(finishCalls.length, 2); + assert.deepEqual(finishCalls[1], finishCalls[0]); assert.equal(finished.terminalReason, "saturated"); assert.equal(finished.manifestPath, manifestPath); diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs b/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs index e6cea4f54..960a30d0d 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs +++ b/plugins/codex-security/mcp-app/tests/test_workbench_state_fallback.mjs @@ -29,6 +29,7 @@ async function testWorkbenchStateFallback() { await writeFakePython(fakePythonPath); await build({ bundle: true, + nodePaths: [fileURLToPath(new URL("../node_modules", import.meta.url))], define: { "import.meta.url": "__filename" }, entryPoints: [path.join(mcpAppRoot, "main.ts")], external: ["fsevents"], diff --git a/plugins/codex-security/mcp-app/tsconfig.json b/plugins/codex-security/mcp-app/tsconfig.json index ef0922f6b..5f455476e 100644 --- a/plugins/codex-security/mcp-app/tsconfig.json +++ b/plugins/codex-security/mcp-app/tsconfig.json @@ -5,6 +5,10 @@ "module": "ESNext", "moduleResolution": "Bundler", "noEmit": true, + "paths": { + "@openai/codex-sdk": ["./node_modules/@openai/codex-sdk"], + "smol-toml": ["./node_modules/smol-toml/dist/index"] + }, "resolveJsonModule": true, "skipLibCheck": true, "strict": true, diff --git a/plugins/codex-security/native/proof-policy-windows.mts b/plugins/codex-security/native/proof-policy-windows.mts index 5ca186a39..65235e6af 100644 --- a/plugins/codex-security/native/proof-policy-windows.mts +++ b/plugins/codex-security/native/proof-policy-windows.mts @@ -8,10 +8,11 @@ import { nativeTarget } from "./platform.mjs"; const testDirectory = join(output, "policy-proof"); const helper = join(testDirectory, "helpers.cjs"); if (process.argv[2] === "build") { + const sdkModules = join(root, "../../../sdk/typescript/node_modules"); execFileSync( process.execPath, [ - join(root, "../../../sdk/typescript/node_modules/esbuild/bin/esbuild"), + join(sdkModules, "esbuild/bin/esbuild"), join(root, "../mcp-app/helpers-main.ts"), "--bundle", "--platform=node", @@ -20,7 +21,11 @@ if (process.argv[2] === "build") { "--define:import.meta.url=__filename", `--outfile=${helper}`, ], - { stdio: "inherit" }, + { + stdio: "inherit", + // Native CI installs the helper's dependencies only in the SDK. + env: { ...process.env, NODE_PATH: sdkModules }, + }, ); const nativeDirectory = join(testDirectory, "native", nativeTarget); mkdirSync(nativeDirectory, { recursive: true }); diff --git a/plugins/codex-security/references/core-scan.md b/plugins/codex-security/references/core-scan.md index 9eb77afe0..75ecefd74 100644 --- a/plugins/codex-security/references/core-scan.md +++ b/plugins/codex-security/references/core-scan.md @@ -9,7 +9,7 @@ Perform one complete, evidence-backed security audit of the exact supplied repos 3. While the baseline runs, read `threat-model.md` once and obtain its independent architecture review within the available worker allowance. Verify its resource rows against their actual consumers, use the returned canonical `threatModel` as the generated model, and build source-backed investigation packets from it. Carry that object and its evidence into the final result instead of reconstructing a shorter summary. Preserve any user-supplied threat model unchanged as the authoritative security assumptions; map its real surfaces and controls without replacing it. 4. Group related source-backed security questions into investigation packets. Each group shares its plausible attacker, protected asset, entry points, expected controls, sensitive operations, component relationships, and actual repository-relative source anchors. Keep each question concrete, preserve distinct attacker boundaries and security mechanisms, and let investigators establish the detailed dataflow. 5. Launch focused investigator subagents with `fork_turns: "none"` as soon as useful packet groups exist. Choose their number and assignments from the amount, complexity, and independence of source-backed work, bounded by the supplied available subagent allowance; use fewer for related packets and more only when distinct surfaces justify them. Keep mapping other surfaces while they run. Send each only its focused-investigator prompt below, assigned packets, investigator perspective, repository path, authorized scope, any supplied scoped-source inventory, exact user context, supplied threat model, applicable packet-specific security guidance and its resolver command, the optional authoritative knowledge-base location, and verified search command. Do not include this reference or another worker's prompt. Supporting code may be outside a requested path, but an affected entry point, control, or operation must be in scope. -6. Before combining or revalidating any returned baseline or investigator result, persist it through the caller's bound `record_codex_security_scan_draft` tool when available, using `complete: false` and partial coverage. Give each candidate a stable `candidateId`; put candidates awaiting parent validation in `coverage.deferred` with a meaningful reason and their original finding payload under `candidate`. Preserve returned counterevidence and unresolved questions too. Checkpoint again after each validation decision, without waiting for other workers or the final report. Put source-validated findings in `findings` with the same `provenance.candidateId`; for a rejection, retain the candidate ID, original evidence, and source-backed counterevidence on a `rejected` coverage surface. An unfinished scan must retain its saved findings and pending candidates without presenting pending work as validated. Reconcile source coverage before combining findings. Union only the baseline and focused investigators' `fully_reviewed_files` with files the parent fully security-audited, then intersect that set with the supplied authorized inventory or an inventory of the selected current scope. Architecture mapping alone and supporting files outside that inventory do not count toward completed audit coverage. Finish the remaining in-scope files in coherent groups, reusing available investigators within the same allowance. Inspect implementation-owning generated or compressed code as data. Do not add overlapping worker counts or claim that a search hit completed a file. Keep this one transient set; do not create a separate progress ledger or receipt format. If a user limit or unavailable source prevents completion, identify the actual remaining paths and report partial coverage. Then combine baseline and investigator findings once. Group observations only when they share the same broken security control and effective remediation; preserve every affected route, operation, sink, and supporting source location. Never merge different security failures solely because they share a CWE. +6. Before combining or revalidating any returned baseline or investigator result, persist it through the caller's bound `record_codex_security_scan_draft` tool when available, using `complete: false` and partial coverage. Give each candidate a stable `candidateId`; put candidates awaiting parent validation in `coverage.deferred` with a meaningful reason and their original finding payload under `candidate`. Preserve returned counterevidence and unresolved questions too. Checkpoint again after each validation decision, without waiting for other workers or the final report. Reuse the saved candidate's `candidateId` (or its existing deferred `id`) when recording a validation decision. Put source-validated findings in `findings` with that `provenance.candidateId`; for a rejection, set the same `candidateId` on a `rejected` coverage surface and retain original evidence and source-backed counterevidence. A prose surface summary alone does not resolve a different candidate. Later drafts preserve omitted findings and unresolved candidates. An unfinished scan must retain its saved findings and pending candidates without presenting pending work as validated. Reconcile source coverage before combining findings. Union only the baseline and focused investigators' `fully_reviewed_files` with files the parent fully security-audited, then intersect that set with the supplied authorized inventory or an inventory of the selected current scope. Architecture mapping alone and supporting files outside that inventory do not count toward completed audit coverage. Finish the remaining in-scope files in coherent groups, reusing available investigators within the same allowance. Inspect implementation-owning generated or compressed code as data. Do not add overlapping worker counts or claim that a search hit completed a file. Keep this one transient set; do not create a separate progress ledger or receipt format. If a user limit or unavailable source prevents completion, identify the actual remaining paths and report partial coverage. Then combine baseline and investigator findings once. Group observations only when they share the same broken security control and effective remediation; preserve every affected route, operation, sink, and supporting source location. Never merge different security failures solely because they share a CWE. 7. Independently validate each unique finding against local source once. Establish its attacker, entry point, trust boundary, attacker-controlled dataflow, transformations, broken control, sensitive operation, prerequisites, effective mitigations, strongest counterevidence, and concrete impact. Record concise, source-backed `rootCause.summary`, `validation.summary`, `attackPath.dataflow.summary`, and `attackPath.reachability.summary` alongside their supporting facts; determine impact, likelihood, and severity from those established facts. State optional configuration, dependency-version, or deployment prerequisites; do not require proof of a real deployment or runtime reproduction. A public library or parser boundary is sufficient when callers control the input. Reject only with source-backed counterevidence, preserve valid baseline findings, record material unresolved proof gaps, and apply the severity rules below. 8. Assemble complete semantic `scope`, `threatModel`, `findings`, and `coverage` using the plugin's `examples/completed-scan/` and `schemas/` as shape references, never as values to copy. Use the canonical field mapping and scenario reconciliation in `threat-model.md`, preserving supplied models unchanged and retaining source-backed architecture, capability, deployment, and uncertainty facts. Give each finding a stable lowercase vulnerability-family `ruleId`, its precise `taxonomy.category` and `taxonomy.cwe` values, genuine `provenance.source`, an instance when separately reported findings would otherwise collide, a `root_control` location when identifiable, all materially affected locations, calibrated severity and rationale, confidence and rationale, verified nonempty source evidence, attacker-to-sink reachability, and practical remediation. Write source evidence as `codeEvidence` entries with required `id`, `label`, `path`, `startLine`, `code`, and `explanation` fields; `endLine`, `language`, and `role` are optional. Use `code`, never `snippet`, and write new root-cause details as `rootCause`, never `root_cause`. Follow `finding-detail-fields.md` when constructing rich finding details. Use actual coverage surface labels and dispositions; report reviewed surfaces, explicit exclusions, deferred work, and unresolved questions honestly, and mark coverage `complete` only when the requested source scope was actually reviewed. Preserve every genuine finding, evidence item, user-supplied assumption, and unresolved proof gap in the caller's complete semantic result. diff --git a/plugins/codex-security/references/scan-contract.md b/plugins/codex-security/references/scan-contract.md index 5b449af8c..3ccd770a5 100644 --- a/plugins/codex-security/references/scan-contract.md +++ b/plugins/codex-security/references/scan-contract.md @@ -106,7 +106,7 @@ Use CWE taxonomy separately. Do not include file names, line numbers, scan IDs, `coverage.json` records scan scope and completion information. Standard and diff summaries also describe reviewed surfaces and outstanding work. -For a Deep parent scan, the host copies the configured paths into `includePaths` and `excludePaths` and sets `completeness` from the coordinator's outcome. A successful aggregate uses `complete`; `surfaces`, `explicitExclusions`, and `deferred` are empty arrays, and `openQuestions` is omitted. If the configured time limit expires before any review completes, the coordinator writes `partial` and records the explanation in `deferred`. Stopped outcomes follow the [stopped-result recovery rules](#stopped-result-recovery). +For a Deep parent scan, the host copies the configured paths into `includePaths` and `excludePaths`. Deep v2 aggregates preserve accepted source coverage in `surfaces`, `explicitExclusions`, `deferred`, `openQuestions`, and `reviews`, including recorded source provenance and receipt references. `completeness` is `partial` when an accepted source is partial, work is deferred, or a surface has disposition `needs_follow_up`; otherwise it is `unknown` when source coverage is unknown, and `complete` when neither condition applies. A successful aggregate or a sealed `completed` scan can therefore retain partial or unknown coverage. Report canonical coverage separately from lifecycle completion. If the configured time limit expires before any review completes, the coordinator writes `partial` and records the explanation in `deferred`. Stopped outcomes follow the [stopped-result recovery rules](#stopped-result-recovery). Each Deep worker writes an ordinary Standard result, including its own coverage. A reducer submits `record_codex_security_deep_reduction({ scanId, findings, scope?, threatModel? })`; its saved results and checkpoints contain the accepted findings and optional scope and threat-model context. diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index 853094a0a..5b4304cdd 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -19,15 +19,30 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from deep_scan_config import resolve_deep_scan_config from filesystem_identity import serialize_filesystem_identity -from finalize_scan_contract import _read_scan_local_json +from finalize_scan_contract import ( + _read_scan_local_json, + open_scan_local_file_descriptor, + write_scan_local_bytes, +) from workbench.handoff import require_current_continuation +from workbench_saved_results import ( + _restore_published_outputs, + _snapshot_published_outputs, + _worker_checkpoint_head, +) +from workbench_scan_usage import capture_scan_usage_owner from workbench_target import ( directory_content_digest, directory_snapshot_regular_file_count, git_revision, worktree_content_digest, ) -from workbench_validation import optional_text, require_uuid, user_context_argument +from workbench_validation import ( + optional_text, + reject_non_finite_json, + require_uuid, + user_context_argument, +) DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup") DEEP_SCAN_WORKER_STATUSES = ("queued", "running", "succeeded", "failed", "canceled") @@ -37,7 +52,12 @@ "invalid_discovery_artifacts", ) DEEP_SCAN_TERMINAL_REASONS = ("saturated", "capped") -DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v1" +DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v2" +SUPPORTED_DEEP_SCAN_WORKFLOWS = { + DEEP_SCAN_WORKFLOW_VERSION, + "deep-scan-mcp/v1", + "deep-security-scan/v1", +} DEEP_SCAN_COORDINATOR_LEASE_SECONDS = 30 DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS = 120 DEEP_SCAN_MAX_ERROR_LENGTH = 2400 @@ -264,20 +284,50 @@ def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlit return row +def require_supported_deep_scan(run: sqlite3.Row) -> None: + if run["schema_version"] != 1 or run["workflow_version"] not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit( + "This Deep Scan uses an unsupported workflow or schema version. " + "Resume it with a compatible Codex Security release." + ) + + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + run["workflow_version"] != "deep-security-scan/v2" + or not isinstance(finalization, dict) + or finalization.get("version") != 1 + ): + raise SystemExit("This Deep Scan uses an unsupported finalization input version.") + + +def deep_scan_finalization_input(run: sqlite3.Row) -> dict[str, Any] | None: + if "finalization_input_json" not in run.keys() or run["finalization_input_json"] is None: + return None + return json.loads(run["finalization_input_json"]) + + def deep_scan_deadline_reached(run: sqlite3.Row) -> bool: elapsed = _parse_timestamp(now()) - _parse_timestamp(str(run["created_at"])) return elapsed.total_seconds() / 3600 >= run["max_time_hours"] +def find_supported_deep_scan_run( + connection: sqlite3.Connection, scan_id: str +) -> sqlite3.Row | None: + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if run is not None: + require_supported_deep_scan(run) + return run + + def require_deep_scan_ready_for_parent_completion( connection: sqlite3.Connection, scan: sqlite3.Row ) -> None: if scan["mode"] != "deep": return - run = connection.execute( - "SELECT status, manifest_path FROM deep_scan_runs WHERE scan_id = ?", - (scan["id"],), - ).fetchone() + run = find_supported_deep_scan_run(connection, scan["id"]) if run is None or run["status"] != "succeeded" or run["manifest_path"] is None: raise SystemExit( "Deep Scan discovery orchestration must finish and persist its manifest before " @@ -410,20 +460,35 @@ def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]: def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: + if connection.in_transaction: + return _deep_scan_state(connection, scan_id) + connection.execute("BEGIN") + try: + state = _deep_scan_state(connection, scan_id) + connection.commit() + return state + except BaseException: + connection.rollback() + raise + + +def _deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]: run = require_deep_scan_run(connection, scan_id) scan = require_scan(connection, run["scan_id"]) worker_rows = connection.execute( """ - SELECT * - FROM deep_scan_workers - WHERE scan_id = ? - ORDER BY created_at, id + SELECT workers.*, attempts.accepted_result_path + FROM deep_scan_workers AS workers + LEFT JOIN deep_scan_attempts AS attempts + ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt + WHERE workers.scan_id = ? + ORDER BY workers.created_at, workers.id """, (run["scan_id"],), ) input_rows = connection.execute( """ - SELECT dedup_worker_id, discovery_worker_id, input_order + SELECT * FROM deep_scan_dedup_inputs WHERE scan_id = ? ORDER BY dedup_worker_id, input_order @@ -453,10 +518,22 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "scanId": run["scan_id"], "targetPath": scan["target_path"], "scope": scan["scope"], - "userContext": scan["user_context"], + "model": scan["model"], + "reasoningEffort": scan["reasoning_effort"], + "userContext": ( + run["discovery_user_context"] + if "discovery_user_context" in run.keys() + else scan["user_context"] + ), "scanDir": scan["scan_dir"], "schemaVersion": run["schema_version"], "workflowVersion": run["workflow_version"], + "finalizationInput": deep_scan_finalization_input(run), + "usageOwner": ( + json.loads(run["usage_owner_json"]) + if "usage_owner_json" in run.keys() and run["usage_owner_json"] + else None + ), "coordinatorGeneration": run["coordinator_generation"], "status": run["status"], "phase": run["phase"], @@ -481,11 +558,55 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "updatedAt": run["updated_at"], "completedAt": run["completed_at"], "workers": [deep_scan_worker_state(row) for row in worker_rows], + "attempts": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "status": row["status"], + "startedAt": row["started_at"], + "completedAt": row["completed_at"], + "endReason": row["end_reason"], + "error": row["error_message"], + "acceptedResultPath": row["accepted_result_path"], + "acceptedResultSha256": row["accepted_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempts WHERE scan_id = ? ORDER BY worker_id, attempt", + (scan_id,), + ) + ], + "attemptSessions": [ + { + "workerId": row["worker_id"], + "attempt": row["attempt"], + "sdkThreadId": row["sdk_thread_id"], + "observedAt": row["observed_at"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_attempt_sessions WHERE scan_id = ? " + "ORDER BY observed_at, worker_id, attempt, sdk_thread_id", + (scan_id,), + ) + ], + "mergeClaims": [ + { + "workerId": row["worker_id"], + "previousWorkerId": row["previous_worker_id"], + "previousResultPath": row["previous_result_path"], + "previousResultSha256": row["previous_result_sha256"], + } + for row in connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE scan_id = ? ORDER BY rowid", (scan_id,) + ) + ], "dedupInputs": [ { "dedupWorkerId": row["dedup_worker_id"], "discoveryWorkerId": row["discovery_worker_id"], "inputOrder": row["input_order"], + "resultManifestPath": row["result_manifest_path"], + "resultManifestSha256": row["result_manifest_sha256"], + "attempt": row["attempt"], } for row in input_rows ], @@ -533,6 +654,9 @@ def deep_scan_worker_state(row: sqlite3.Row) -> dict[str, Any]: "promptPath": row["prompt_path"], "artifactDir": row["artifact_dir"], "resultManifestPath": row["result_manifest_path"], + "acceptedResultPath": row["accepted_result_path"] + if "accepted_result_path" in row.keys() + else None, "attempt": row["attempt"], "sdkThreadId": row["sdk_thread_id"], "completionSequence": row["completion_sequence"], @@ -561,30 +685,83 @@ def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | floa return resolve_deep_scan_config(available_parallelism) +def read_deep_scan_execution_settings(scan_dir: Path) -> dict[str, Any]: + relative_path = "artifacts/deep_discovery/execution-settings.json" + if not (scan_dir / relative_path).exists(): + raise SystemExit( + "This Deep Scan has no recorded original execution settings; " + "its executable and Codex home cannot be recovered." + ) + saved = _read_scan_local_json(scan_dir, relative_path, "Deep Scan execution settings") + if saved.get("version") != 1: + raise SystemExit("This Deep Scan uses an unsupported execution settings version.") + settings = saved.get("settings") + if not isinstance(settings, dict) or not all( + isinstance(settings.get(key), str) for key in ("codexPath", "codexHome") + ): + raise SystemExit( + "Deep Scan execution settings are missing the recorded executable or Codex home." + ) + return settings + + def ensure_deep_scan_run( connection: sqlite3.Connection, scan: sqlite3.Row, config: dict[str, int | float], workflow_version: str, timestamp: str, + args: argparse.Namespace | None = None, ) -> sqlite3.Row: existing = connection.execute( "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) ).fetchone() if existing is not None: + require_supported_deep_scan(existing) return existing if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") if scan["status"] != "running": raise SystemExit("Only a running Deep Scan can start orchestration.") + if args is not None and hasattr(args, "execution_settings"): + # The creation transaction serializes contenders. Persist before the run + # becomes recoverable; joined callers never replace the original bytes. + scan_dir = Path(scan["scan_dir"]) + relative_path = "artifacts/deep_discovery/execution-settings.json" + if (scan_dir / relative_path).exists(): + saved = read_deep_scan_execution_settings(scan_dir) + # A managed scan directory can survive a rolled-back creation. + # Keep its saved selections and the returned scan model consistent. + connection.execute( + "UPDATE scans SET model = ?, reasoning_effort = ? WHERE id = ?", + ( + saved.get("model"), + saved.get("reasoningEffort"), + scan["id"], + ), + ) + else: + if args.execution_settings is None: + raise SystemExit("The original Deep Scan execution settings could not be captured.") + settings = dict(args.execution_settings) + # The persisted scan remains authoritative when the caller did not + # supply a model or effort (for example a managed handoff). + for key, column in (("model", "model"), ("reasoningEffort", "reasoning_effort")): + if scan[column] is not None: + settings[key] = scan[column] + write_scan_local_bytes( + scan_dir, + relative_path, + (json.dumps({"version": 1, "settings": settings}, indent=2) + "\n").encode(), + ) connection.execute( """ INSERT INTO deep_scan_runs ( scan_id, schema_version, workflow_version, status, phase, workers, subagents, stop_after_no_new, stop_after_consecutive_errors, - max_discovery_runs, max_time_hours, + max_discovery_runs, max_time_hours, discovery_user_context, created_at, updated_at - ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( scan["id"], @@ -595,11 +772,19 @@ def ensure_deep_scan_run( config["stopAfterConsecutiveErrors"], config["maxDiscoveryRuns"], config["maxTimeHours"], + scan["user_context"], timestamp, timestamp, ), ) - return require_deep_scan_run(connection, scan["id"]) + run = require_deep_scan_run(connection, scan["id"]) + if "usage_owner_json" in run.keys(): + connection.execute( + "UPDATE deep_scan_runs SET usage_owner_json = ? WHERE scan_id = ?", + (json.dumps(capture_scan_usage_owner(connection, scan)), scan["id"]), + ) + run = require_deep_scan_run(connection, scan["id"]) + return run def existing_deep_scan_for_target( @@ -687,6 +872,11 @@ def begin_deep_scan_for_scan( ) -> dict[str, Any]: scan_id = require_uuid(scan_id, "scan-id") candidate = require_scan(connection, scan_id) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_supported_deep_scan(existing) workspace = require_workspace(connection, candidate["workspace_id"]) if ( candidate["mode"] == "deep" @@ -724,23 +914,10 @@ def begin_deep_scan_for_scan( ) if scan["mode"] != "deep": raise SystemExit("Deep Scan orchestration requires a scan in deep mode.") - model = optional_text(args.model, maximum=200) - reasoning_effort = optional_text(args.reasoning_effort, maximum=32) - if model is not None or reasoning_effort is not None: - connection.execute( - """ - UPDATE scans - SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) - WHERE id = ? - """, - (model, reasoning_effort, scan_id), - ) - connection.commit() - existing = connection.execute( - "SELECT scan_id FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) - ).fetchone() if existing is not None: return deep_scan_result(connection, scan_id, start_disposition="joined") + model = optional_text(args.model, maximum=200) + reasoning_effort = optional_text(args.reasoning_effort, maximum=32) config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: @@ -753,7 +930,24 @@ def begin_deep_scan_for_scan( args.claim_token, error_message="Deep Scan orchestration is owned by another continuation.", ) - ensure_deep_scan_run(connection, scan, config, workflow_version, now()) + existing = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone() + if existing is not None: + require_supported_deep_scan(existing) + connection.commit() + return deep_scan_result(connection, scan_id, start_disposition="joined") + if model is not None or reasoning_effort is not None: + connection.execute( + """ + UPDATE scans + SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort) + WHERE id = ? + """, + (model, reasoning_effort, scan_id), + ) + scan = require_scan(connection, scan_id) + ensure_deep_scan_run(connection, scan, config, workflow_version, now(), args) connection.commit() except BaseException: connection.rollback() @@ -788,14 +982,16 @@ def begin_deep_scan_for_target( existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) if existing is not None: existing_run = connection.execute( - "SELECT 1 FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) ).fetchone() + if existing_run is not None: + require_supported_deep_scan(existing_run) if existing_run is None: config = effective_deep_scan_config(args) workflow_version = optional_text(args.workflow_version, maximum=256) if workflow_version is None: raise SystemExit("workflow-version is required.") - ensure_deep_scan_run(connection, existing, config, workflow_version, now()) + ensure_deep_scan_run(connection, existing, config, workflow_version, now(), args) connection.commit() return deep_scan_result( connection, @@ -915,7 +1111,7 @@ def begin_deep_scan_for_target( (scan_id, timestamp, workspace_id), ) scan = require_scan(connection, scan_id) - ensure_deep_scan_run(connection, scan, config, workflow_version, timestamp) + ensure_deep_scan_run(connection, scan, config, workflow_version, timestamp, args) connection.commit() except BaseException: connection.rollback() @@ -924,6 +1120,9 @@ def begin_deep_scan_for_target( def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: + workflow_version = optional_text(args.workflow_version, maximum=256) + if workflow_version not in SUPPORTED_DEEP_SCAN_WORKFLOWS: + raise SystemExit("This Deep Scan uses an unsupported workflow version.") thread_id = optional_text(args.thread_id, maximum=512) if thread_id is None: raise SystemExit("thread-id is required.") @@ -971,6 +1170,8 @@ def coordinator_lease_is_live( try: heartbeat = json.loads(heartbeat_path.read_text(encoding="utf-8")) if heartbeat["coordinatorGeneration"] == run["coordinator_generation"]: + if heartbeat.get("released") is True: + return False heartbeat_time = max(heartbeat_time, _parse_timestamp(heartbeat["updatedAt"])) except (OSError, KeyError, TypeError, ValueError): pass @@ -979,6 +1180,7 @@ def coordinator_lease_is_live( def require_current_coordinator(run: sqlite3.Row, args: argparse.Namespace) -> None: + require_supported_deep_scan(run) generation = getattr(args, "coordinator_generation", None) if run["coordinator_generation"] == 1: if generation is not None: @@ -1009,7 +1211,25 @@ def claim_deep_scan_coordinator_locked( args.claim_token, error_message="Deep Scan orchestration is owned by another continuation.", ) - run, _ = require_running_deep_scan(connection, scan_id) + run = require_deep_scan_run(connection, scan_id) + require_supported_deep_scan(run) + selected_parent = ( + run["status"] == "succeeded" and deep_scan_finalization_input(run) is not None + ) + if selected_parent and scan["status"] == "complete": + connection.commit() + return { + **deep_scan_result(connection, scan_id), + "coordinatorDisposition": "observing", + } + if not selected_parent: + run, _ = require_running_deep_scan(connection, scan_id) + elif ( + run["cancel_requested"] + or scan["status"] != "running" + or scan["canceled_at"] is not None + ): + raise SystemExit("Only a running scan can complete its selected Deep Scan result.") timestamp = now() if args.coordinator_generation is not None: require_current_coordinator(run, args) @@ -1022,15 +1242,25 @@ def claim_deep_scan_coordinator_locked( } else: adopted = run["coordinator_generation"] > 1 or run["phase"] != "setup" - if adopted: - recover_expired_coordinator(connection, run, timestamp) disposition = "adopted" if adopted else "claimed" + if ( + getattr(args, "require_execution_settings", False) + and deep_scan_finalization_input(run) is None + ): + scan_dir = Path(scan["scan_dir"]) + if ( + run["workflow_version"] == DEEP_SCAN_WORKFLOW_VERSION + or (scan_dir / "artifacts/deep_discovery/execution-settings.json").exists() + ): + read_deep_scan_execution_settings(scan_dir) + if disposition == "adopted": + recover_expired_coordinator(connection, run, timestamp) connection.execute( """ UPDATE deep_scan_runs SET coordinator_generation = coordinator_generation + ?, updated_at = ? - WHERE scan_id = ? AND status = 'running' + WHERE scan_id = ? AND status IN ('running', 'succeeded') """, (int(args.coordinator_generation != run["coordinator_generation"]), timestamp, scan_id), ) @@ -1047,6 +1277,9 @@ def claim_deep_scan_coordinator_locked( def recover_expired_coordinator( connection: sqlite3.Connection, run: sqlite3.Row, timestamp: str ) -> None: + require_supported_deep_scan(run) + if deep_scan_finalization_input(run) is not None: + return scan_id = run["scan_id"] recover_candidate_ledger_publication(connection, scan_id) legacy_generation = int(run["coordinator_generation"] == 1) @@ -1173,6 +1406,7 @@ def require_running_deep_scan( connection: sqlite3.Connection, scan_id: str ) -> tuple[sqlite3.Row, sqlite3.Row]: run = require_deep_scan_run(connection, scan_id) + require_supported_deep_scan(run) scan = require_scan(connection, run["scan_id"]) if run["status"] != "running" or run["cancel_requested"]: raise SystemExit("Only a running Deep Scan can update orchestration state.") @@ -1193,6 +1427,132 @@ def require_worker_transition(current: str, requested: str) -> None: raise SystemExit(f"Deep Scan worker cannot transition from {current} to {requested}.") +def snapshot_accepted_result(scan: sqlite3.Row, worker: sqlite3.Row) -> tuple[str, str]: + source = deep_scan_path( + scan, worker["result_manifest_path"], "Accepted worker result", kind="file" + ) + scan_dir = Path(scan["scan_dir"]) + contents = Path(source).read_bytes() + semantic = json.loads(contents) + if isinstance(semantic, dict): + semantic.pop("handoffClaimToken", None) + directory = Path(worker["artifact_dir"]) / "checkpoints" + head = ( + _worker_checkpoint_head( + scan_dir, Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix(), scan["id"] + ) + if worker["kind"] == "discovery" + else None + ) + candidates = [scan_dir / head] if head else sorted(directory.glob("*.json")) + for checkpoint in candidates: + safe = deep_scan_path(scan, str(checkpoint), "Accepted worker checkpoint", kind="file") + checkpoint_bytes = Path(safe).read_bytes() + if json.loads(checkpoint_bytes) == semantic: + return safe, hashlib.sha256(checkpoint_bytes).hexdigest() + if head: + raise SystemExit( + "The accepted worker result does not match its current checkpoint head." + ) + # Legacy/direct file producers may have no checkpoint. Use the existing native + # checkpoint store; typed artifact writers already supplied the matching copy. + digest = hashlib.sha256(contents).hexdigest() + destination = directory / f"{digest}.json" + if destination.exists(): + raise SystemExit("An existing worker checkpoint does not match its accepted content.") + write_scan_local_bytes(scan_dir, destination.relative_to(scan_dir).as_posix(), contents) + return str(destination), digest + + +def record_worker_attempt( + connection: sqlite3.Connection, + scan: sqlite3.Row, + worker: sqlite3.Row, + timestamp: str, + *, + observed_thread_id: str | None = None, + error: str | None = None, + end_reason: str | None = None, +) -> None: + if worker["status"] == "queued" or worker["attempt"] < 1: + return + connection.execute( + """ + UPDATE deep_scan_attempts + SET status = 'replaced', completed_at = ?, end_reason = 'replacement_attempt' + WHERE worker_id = ? AND attempt < ? AND completed_at IS NULL + """, + (timestamp, worker["id"], worker["attempt"]), + ) + status = worker["status"] + if end_reason in DEEP_SCAN_REPLACEABLE_FAILURE_KINDS: + status = "failed" + if status != "running": + error = worker["error_message"] + if status == "running" and error: + status = "failed" + completed = timestamp if status != "running" else None + reason = end_reason or ( + "execution_or_artifact_error" if status == "failed" else status if completed else None + ) + accepted_path = accepted_sha = None + if status == "succeeded" and worker["result_manifest_path"]: + accepted_path, accepted_sha = snapshot_accepted_result(scan, worker) + connection.execute( + """ + INSERT INTO deep_scan_attempts ( + scan_id, worker_id, attempt, status, started_at, completed_at, + end_reason, error_message, accepted_result_path, accepted_result_sha256 + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(worker_id, attempt) DO UPDATE SET + status = CASE WHEN deep_scan_attempts.completed_at IS NULL THEN excluded.status + ELSE deep_scan_attempts.status END, + completed_at = COALESCE(deep_scan_attempts.completed_at, excluded.completed_at), + end_reason = COALESCE(deep_scan_attempts.end_reason, excluded.end_reason), + error_message = COALESCE(excluded.error_message, deep_scan_attempts.error_message), + accepted_result_path = COALESCE(excluded.accepted_result_path, + deep_scan_attempts.accepted_result_path), + accepted_result_sha256 = COALESCE(excluded.accepted_result_sha256, + deep_scan_attempts.accepted_result_sha256) + """, + ( + scan["id"], + worker["id"], + worker["attempt"], + status, + timestamp, + completed, + reason, + error, + accepted_path, + accepted_sha, + ), + ) + if observed_thread_id: + connection.execute( + """ + INSERT OR IGNORE INTO deep_scan_attempt_sessions ( + scan_id, worker_id, attempt, sdk_thread_id, observed_at + ) VALUES (?, ?, ?, ?, ?) + """, + (scan["id"], worker["id"], worker["attempt"], observed_thread_id, timestamp), + ) + + +def worker_result_reference( + connection: sqlite3.Connection, scan: sqlite3.Row, worker: sqlite3.Row +) -> tuple[str, str]: + accepted = connection.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE worker_id = ? AND attempt = ?", + (worker["id"], worker["attempt"]), + ).fetchone() + if accepted is not None and accepted["accepted_result_path"]: + return accepted["accepted_result_path"], accepted["accepted_result_sha256"] + # Old accepted workers have no attempt history; freeze their current accepted result on claim. + return snapshot_accepted_result(scan, worker) + + def upsert_deep_scan_worker( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: @@ -1236,11 +1596,14 @@ def upsert_deep_scan_worker( scan, args.artifact_dir, "Worker artifact directory", kind="directory" ) result_manifest_path = ( - deep_scan_path( - scan, - args.result_manifest_path, - "Worker result manifest path", - kind="file", + ( + deep_scan_output_path( + scan, args.result_manifest_path, "Worker result manifest path" + ) + if terminal_repeat + else deep_scan_path( + scan, args.result_manifest_path, "Worker result manifest path", kind="file" + ) ) if args.result_manifest_path else None @@ -1299,8 +1662,18 @@ def upsert_deep_scan_worker( timestamp, ), ) + record_worker_attempt( + connection, + scan, + require_deep_scan_worker(connection, worker_id), + timestamp, + observed_thread_id=optional_text(args.sdk_thread_id, maximum=512), + error=optional_text(args.error_message, maximum=2400), + end_reason=args.replaceable_failure_kind, + ) + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result if existing["scan_id"] != scan_id or existing["kind"] != args.kind: raise SystemExit("Deep Scan worker identity does not match its persisted run and kind.") @@ -1323,8 +1696,15 @@ def upsert_deep_scan_worker( and repeated_error != existing["error_message"] ): raise SystemExit("Deep Scan worker terminal state is immutable.") + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_attempts WHERE worker_id = ? AND attempt = ?", + (worker_id, existing["attempt"]), + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["workerReceipt"] = json.loads(receipt["receipt_json"]) connection.commit() - return deep_scan_result(connection, scan_id) + return result attempt = args.attempt if args.attempt is not None else existing["attempt"] if attempt < existing["attempt"]: raise SystemExit("Deep Scan worker attempt cannot decrease.") @@ -1420,11 +1800,30 @@ def upsert_deep_scan_worker( worker_id, ), ) + record_worker_attempt( + connection, + scan, + require_deep_scan_worker(connection, worker_id), + timestamp, + observed_thread_id=optional_text(args.sdk_thread_id, maximum=512), + error=optional_text(args.error_message, maximum=2400), + end_reason=args.replaceable_failure_kind, + ) + result = deep_scan_result(connection, scan_id) + if args.status in {"succeeded", "failed", "canceled"}: + receipt = next( + worker for worker in result["deepScan"]["workers"] if worker["id"] == worker_id + ) + result["deepScan"]["workerReceipt"] = receipt + connection.execute( + "UPDATE deep_scan_attempts SET receipt_json = ? WHERE worker_id = ? AND attempt = ?", + (json.dumps(receipt), worker_id, attempt), + ) connection.commit() except BaseException: connection.rollback() raise - return deep_scan_result(connection, scan_id) + return result def claim_deep_scan_dedup( @@ -1437,7 +1836,8 @@ def claim_deep_scan_dedup( raise SystemExit("Dedup input worker IDs must be unique.") connection.execute("BEGIN IMMEDIATE") try: - run, scan = require_running_deep_scan(connection, scan_id) + run = require_deep_scan_run(connection, scan_id) + scan = require_scan(connection, scan_id) require_current_coordinator(run, args) prompt_path = deep_scan_path(scan, args.prompt_path, "Dedup prompt path", kind="file") artifact_dir = deep_scan_path( @@ -1466,9 +1866,11 @@ def claim_deep_scan_dedup( and existing["artifact_dir"] == artifact_dir and persisted_inputs == input_ids ): + result = deep_scan_result(connection, scan_id) connection.commit() - return deep_scan_result(connection, scan_id) + return result raise SystemExit("Dedup worker ID is already used by a different reducer claim.") + require_running_deep_scan(connection, scan_id) active_reducer = connection.execute( """ SELECT 1 FROM deep_scan_workers @@ -1536,14 +1938,41 @@ def claim_deep_scan_dedup( """, (worker_id, scan_id, prompt_path, artifact_dir, timestamp, timestamp), ) + previous = connection.execute( + "SELECT * FROM deep_scan_workers WHERE scan_id = ? AND kind = 'dedup' " + "AND status = 'succeeded' ORDER BY completed_at DESC, rowid DESC LIMIT 1", + (scan_id,), + ).fetchone() + previous_path, previous_sha = ( + worker_result_reference(connection, scan, previous) if previous else (None, None) + ) + connection.execute( + """ + INSERT INTO deep_scan_merge_claims ( + worker_id, scan_id, previous_worker_id, previous_result_path, previous_result_sha256 + ) VALUES (?, ?, ?, ?, ?) + """, + (worker_id, scan_id, previous["id"] if previous else None, previous_path, previous_sha), + ) for input_order, input_id in enumerate(input_ids): + discovery = require_deep_scan_worker(connection, input_id) + accepted_path, accepted_sha = worker_result_reference(connection, scan, discovery) connection.execute( """ INSERT INTO deep_scan_dedup_inputs ( - scan_id, dedup_worker_id, discovery_worker_id, input_order - ) VALUES (?, ?, ?, ?) + scan_id, dedup_worker_id, discovery_worker_id, input_order, + result_manifest_path, result_manifest_sha256, attempt + ) VALUES (?, ?, ?, ?, ?, ?, ?) """, - (scan_id, worker_id, input_id, input_order), + ( + scan_id, + worker_id, + input_id, + input_order, + accepted_path, + accepted_sha, + discovery["attempt"], + ), ) connection.execute( f""" @@ -1565,11 +1994,12 @@ def claim_deep_scan_dedup( """, (timestamp, scan_id), ) + result = deep_scan_result(connection, scan_id) connection.commit() except BaseException: connection.rollback() raise - return deep_scan_result(connection, scan_id) + return result def commit_deep_scan_dedup( @@ -1595,8 +2025,14 @@ def commit_deep_scan_dedup_locked( if worker["scan_id"] != scan_id or worker["kind"] != "dedup": raise SystemExit("Dedup worker does not belong to this Deep Scan.") if worker["status"] == "succeeded": + receipt = connection.execute( + "SELECT receipt_json FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + result = deep_scan_result(connection, scan_id) + if receipt is not None and receipt["receipt_json"]: + result["deepScan"]["committedMerge"] = json.loads(receipt["receipt_json"]) connection.commit() - return deep_scan_result(connection, scan_id) + return result require_running_deep_scan(connection, scan_id) if worker["status"] not in {"queued", "running"}: raise SystemExit("Only an active dedup worker can commit a result.") @@ -1642,6 +2078,23 @@ def commit_deep_scan_dedup_locked( ) if not inputs or any(row["merge_state"] != "merging" for row in inputs): raise SystemExit("Dedup inputs are not in the claimed merging state.") + claim = connection.execute( + "SELECT * FROM deep_scan_merge_claims WHERE worker_id = ?", (worker_id,) + ).fetchone() + references = [ + (row["result_manifest_path"], row["result_manifest_sha256"]) + for row in connection.execute( + "SELECT * FROM deep_scan_dedup_inputs WHERE dedup_worker_id = ? ORDER BY input_order", + (worker_id,), + ) + ] + if claim is not None and claim["previous_result_path"]: + references.append((claim["previous_result_path"], claim["previous_result_sha256"])) + for path, digest in references: + if path is not None and digest is not None: + safe_path = deep_scan_path(scan, path, "Claimed reducer input", kind="file") + if hashlib.sha256(Path(safe_path).read_bytes()).hexdigest() != digest: + raise SystemExit("A claimed Deep Scan reducer input changed after acceptance.") if candidate_ledger_path and canonical_candidate_ledger_path: canonical_path = Path(canonical_candidate_ledger_path) publication_copy = canonical_path.with_name( @@ -1685,6 +2138,27 @@ def commit_deep_scan_dedup_locked( """, (no_new_streak, timestamp, scan_id), ) + committed_worker = require_deep_scan_worker(connection, worker_id) + record_worker_attempt( + connection, + scan, + committed_worker, + timestamp, + observed_thread_id=committed_worker["sdk_thread_id"], + ) + accepted_path, accepted_sha = worker_result_reference(connection, scan, committed_worker) + result = deep_scan_result(connection, scan_id) + result["deepScan"]["committedMerge"] = { + "workerId": worker_id, + "resultManifestPath": accepted_path, + "resultManifestSha256": accepted_sha, + "newFindings": args.new_findings_count, + } + connection.execute( + "INSERT INTO deep_scan_merge_claims (worker_id, scan_id, receipt_json) VALUES (?, ?, ?) " + "ON CONFLICT(worker_id) DO UPDATE SET receipt_json = excluded.receipt_json", + (worker_id, scan_id, json.dumps(result["deepScan"]["committedMerge"])), + ) connection.commit() except BaseException: connection.rollback() @@ -1697,10 +2171,20 @@ def commit_deep_scan_dedup_locked( finish_staged_file(promotion) if publication_copy is not None: publication_copy.unlink(missing_ok=True) - return deep_scan_result(connection, scan_id) + return result -def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: +def finish_deep_scan( + connection: sqlite3.Connection, args: argparse.Namespace, select_finalization: bool = False +) -> dict[str, Any]: + if select_finalization: + import sys + + args = argparse.Namespace( + **vars(args), + select_finalization=True, + finalization_result_path=json.load(sys.stdin)["resultPath"], + ) scan_id = require_uuid(args.scan_id, "scan-id") with scan_completion_lock(scan_id): return finish_deep_scan_locked(connection, args, scan_id) @@ -1714,15 +2198,26 @@ def finish_deep_scan_locked( ] if len(set(omitted_worker_ids)) != len(omitted_worker_ids): raise SystemExit("Omitted Deep Scan worker IDs must be unique.") + selecting = getattr(args, "select_finalization", False) promotion: tuple[Path, Path, Path | None] | None = None connection.execute("BEGIN IMMEDIATE") try: run = require_deep_scan_run(connection, scan_id) require_current_coordinator(run, args) scan = require_scan(connection, scan_id) + if selecting and run["workflow_version"] != "deep-security-scan/v2": + raise SystemExit("Selected finalization requires the supported v2 workflow.") + finalization = deep_scan_finalization_input(run) + if finalization is not None and ( + args.terminal_reason != finalization["terminalReason"] + or omitted_worker_ids != finalization["omittedWorkerIds"] + ): + raise SystemExit( + "Deep Scan finalization must retain its selected reason and omissions." + ) manifest_path = ( deep_scan_output_path(scan, args.manifest_path, "Deep Scan coordinator manifest path") - if args.staged_manifest_path + if args.staged_manifest_path or selecting else deep_scan_path( scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file" ) @@ -1732,6 +2227,7 @@ def finish_deep_scan_locked( failure_capped = False if ( standard_scan_manifest + and not selecting and args.terminal_reason == "capped" and (run["status"] == "running" or omitted_worker_ids) ): @@ -1838,7 +2334,10 @@ def finish_deep_scan_locked( "Deep Scan cannot finish capped before reaching its configured maximum." ) canonical_artifacts = None - if standard_scan_manifest: + if selecting: + if not standard_scan_manifest: + raise SystemExit("Selected Deep Scan finalization requires the parent manifest.") + elif standard_scan_manifest: for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"): deep_scan_path( scan, @@ -1942,6 +2441,17 @@ def finish_deep_scan_locked( f"Deep Scan {args.terminal_reason} completion must exactly identify all buffered discovery " "workers with --omitted-worker-id." ) + if selecting: + selection = selected_deep_scan_finalization( + connection, run, scan, args, omitted_worker_ids, zero_discovery_deadline + ) + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, terminal_reason = ?, " + "phase = 'terminal', updated_at = ? WHERE scan_id = ?", + (json.dumps(selection), selection["terminalReason"], now(), scan_id), + ) + connection.commit() + return deep_scan_result(connection, scan_id) if args.staged_manifest_path: staged_manifest_path = deep_scan_path( scan, @@ -1972,6 +2482,158 @@ def finish_deep_scan_locked( return deep_scan_result(connection, scan_id) +def selected_deep_scan_finalization( + connection: sqlite3.Connection, + run: sqlite3.Row, + scan: sqlite3.Row, + args: argparse.Namespace, + omitted_worker_ids: list[str], + zero_discovery_deadline: bool, +) -> dict[str, Any]: + """Select the committed attempt's immutable aggregate before publication.""" + if run["finalization_input_json"] is not None: + return json.loads(run["finalization_input_json"]) + result_path = getattr(args, "finalization_result_path", None) + relative: str | None = None + digest: str | None = None + if result_path is None: + if not zero_discovery_deadline: + raise SystemExit("Deep Scan finalization requires its accepted reducer result.") + else: + accepted = connection.execute( + "SELECT attempts.accepted_result_path, attempts.accepted_result_sha256 " + "FROM deep_scan_workers AS workers LEFT JOIN deep_scan_attempts AS attempts " + "ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt " + "WHERE workers.id = (SELECT id FROM deep_scan_workers WHERE scan_id = ? " + "AND kind = 'dedup' AND status = 'succeeded' ORDER BY completed_at DESC, id DESC LIMIT 1) " + "AND (workers.result_manifest_path = ? OR attempts.accepted_result_path = ?)", + (scan["id"], result_path, result_path), + ).fetchone() + if ( + accepted is None + or not accepted["accepted_result_path"] + or not accepted["accepted_result_sha256"] + ): + raise SystemExit( + "Deep Scan finalization requires its committed accepted reducer reference." + ) + scan_dir = Path(scan["scan_dir"]) + source = Path( + deep_scan_path( + scan, accepted["accepted_result_path"], "Selected Deep Scan result", kind="file" + ) + ) + relative = source.relative_to(scan_dir).as_posix() + digest = accepted["accepted_result_sha256"] + selection = { + "version": 1, + "resultPath": relative, + "resultSha256": digest, + "terminalReason": args.terminal_reason, + "omittedWorkerIds": omitted_worker_ids, + "selectedAt": now(), + } + return selection + + +def budget_unmerged_workers(connection: sqlite3.Connection, scan_id: str) -> list[sqlite3.Row]: + return connection.execute( + "SELECT workers.*, attempts.accepted_result_path, attempts.accepted_result_sha256 " + "FROM deep_scan_workers AS workers JOIN deep_scan_attempts AS attempts " + "ON attempts.worker_id = workers.id AND attempts.attempt = workers.attempt " + "WHERE workers.scan_id = ? AND workers.kind = 'discovery' " + "AND workers.status = 'succeeded' AND workers.merge_state IN ('buffered', 'merging') " + "ORDER BY workers.completion_sequence, workers.id", + (scan_id,), + ).fetchall() + + +def prepare_budget_exhausted_deep_scan( + connection: sqlite3.Connection, + scan: sqlite3.Row, + scan_dir: Path, + warning: str, + write_draft: Callable[[dict[str, Any] | None, list[sqlite3.Row]], None], +) -> None: + """Finish local budget publication without starting or promoting scan work.""" + snapshots = _snapshot_published_outputs(scan_dir) + connection.execute("BEGIN IMMEDIATE") + try: + # The completion lock and this transaction preserve any committed selector. + run, scan = require_running_deep_scan(connection, scan["id"]) + unmerged = budget_unmerged_workers(connection, scan["id"]) + if run["finalization_input_json"] is not None: + selection = json.loads(run["finalization_input_json"]) + else: + reducer = connection.execute( + "SELECT attempts.accepted_result_path FROM deep_scan_workers AS workers " + "LEFT JOIN deep_scan_attempts AS attempts ON attempts.worker_id = workers.id " + "AND attempts.attempt = workers.attempt WHERE workers.scan_id = ? " + "AND workers.kind = 'dedup' AND workers.status = 'succeeded' " + "ORDER BY workers.completed_at DESC, workers.id DESC LIMIT 1", + (scan["id"],), + ).fetchone() + if reducer is not None and not reducer["accepted_result_path"]: + raise SystemExit("Budget completion requires the committed reducer reference.") + selection = selected_deep_scan_finalization( + connection, + run, + scan, + argparse.Namespace( + finalization_result_path=reducer["accepted_result_path"] if reducer else None, + terminal_reason="capped", + ), + [row["id"] for row in unmerged], + reducer is None, + ) + accepted = None + if selection["resultPath"] is not None: + descriptor = open_scan_local_file_descriptor( + scan_dir, selection["resultPath"], "Accepted reducer" + ) + with os.fdopen(descriptor, "rb") as source: + contents = source.read() + if hashlib.sha256(contents).hexdigest() != selection["resultSha256"]: + raise SystemExit("The accepted reducer changed before budget completion.") + accepted = json.loads(contents, parse_constant=reject_non_finite_json) + if ( + not isinstance(accepted, dict) + or accepted.get("scanId") != scan["id"] + or accepted.get("complete", True) is not True + or not isinstance(accepted.get("sourceCoverage"), dict) + ): + raise SystemExit("Budget completion requires the accepted complete reducer.") + write_draft(accepted, unmerged) + timestamp = now() + cancel_active_workers(connection, scan["id"], timestamp) + connection.execute( + "UPDATE deep_scan_runs SET status = 'succeeded', phase = 'terminal', " + "terminal_reason = ?, cancel_requested = 1, error_message = ?, " + "manifest_path = ?, finalization_input_json = ?, completed_at = ?, updated_at = ? " + "WHERE scan_id = ?", + ( + selection["terminalReason"], + warning, + str(scan_dir / "scan-manifest.json"), + json.dumps(selection), + timestamp, + timestamp, + scan["id"], + ), + ) + warnings = json.loads(scan["completion_warnings_json"]) + if warning not in warnings: + connection.execute( + "UPDATE scans SET completion_warnings_json = ? WHERE id = ?", + (json.dumps([*warnings, warning]), scan["id"]), + ) + connection.commit() + except BaseException: + connection.rollback() + _restore_published_outputs(scan_dir, snapshots) + raise + + def fail_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]: scan_id = require_uuid(args.scan_id, "scan-id") with scan_completion_lock(scan_id): @@ -2154,6 +2816,11 @@ def cancel_from_parent_scan(connection: sqlite3.Connection, scan_id: str, timest def cancel_active_workers(connection: sqlite3.Connection, scan_id: str, timestamp: str) -> None: + connection.execute( + "UPDATE deep_scan_attempts SET status = 'canceled', completed_at = ?, " + "end_reason = 'scan_stopped' WHERE scan_id = ? AND completed_at IS NULL", + (timestamp, scan_id), + ) connection.execute( """ UPDATE deep_scan_workers diff --git a/plugins/codex-security/scripts/report_projection.py b/plugins/codex-security/scripts/report_projection.py index 98b4f00d1..73c6b90f3 100644 --- a/plugins/codex-security/scripts/report_projection.py +++ b/plugins/codex-security/scripts/report_projection.py @@ -513,7 +513,14 @@ def _target_scope_lines(target: dict[str, Any]) -> list[str]: def _surface_notes(surface: dict[str, Any]) -> str: - notes = surface.get("notes", "No additional canonical notes were recorded.") + notes = surface.get( + "notes", surface.get("reason", "No additional canonical notes were recorded.") + ) + if "notes" in surface and "reason" in surface and surface["reason"] != notes: + notes += f" {surface['reason']}" + source = _coverage_source(surface) + if source: + notes = f"{source}. {notes}" receipt_refs = surface.get("receiptRefs", []) if not isinstance(receipt_refs, list) or not receipt_refs: return _cell(notes) @@ -523,6 +530,51 @@ def _surface_notes(surface: dict[str, Any]) -> str: return _cell(f"{notes} Evidence: {evidence}") +def _coverage_source(item: dict[str, Any]) -> str: + provenance = item.get("provenance", {}) + if not isinstance(provenance, dict) or not provenance.get("workerId"): + return "" + source = f"Review {provenance['workerId']}" + if provenance.get("attempt") is not None: + source += f", attempt {provenance['attempt']}" + return source + + +def _remediation_section(finding: dict[str, Any]) -> list[str]: + remediation = _text(finding.get("remediation"), "No canonical remediation was recorded.") + lines = ["", "#### Remediation", "", remediation] + seen = {remediation} + sources = finding.get("provenance", {}).get("sourceFindings", []) + originals = ( + [ + source + for source in sources + if isinstance(source, dict) and isinstance(source.get("finding"), dict) + ] + if isinstance(sources, list) + else [] + ) + for source in originals: + text = _text(source["finding"].get("remediation"), "") + if text and text not in seen: + seen.add(text) + lines.extend(["", f"Source {_text(source.get('id'), 'finding')}: {text}"]) + for field, label in ( + ("remediationTests", "Tests"), + ("preventiveControls", "Preventive controls"), + ): + values = list( + dict.fromkeys( + value + for original in [finding, *(source["finding"] for source in originals)] + for value in _strings(original.get(field)) + ) + ) + if values: + lines.extend(["", f"{label}:", *_bullets(values, "None recorded.")]) + return lines + + def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: validation = finding.get("validation") if isinstance(finding.get("validation"), dict) else {} _, raw_root_cause = merged_root_cause(finding) @@ -616,8 +668,6 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: severity.get("changeConditions"), "Additional runtime or deployment evidence could raise or lower this severity.", ) - remediation_tests = _strings(finding.get("remediationTests")) - preventive_controls = _strings(finding.get("preventiveControls")) attack_steps = _strings(attack_path.get("steps")) cwes = ", ".join(finding["taxonomy"]["cwe"]) or "none" title = _text(finding["title"], "Untitled finding") @@ -736,18 +786,7 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]: lines.extend( ["", f"{label} assessment:", *(f"- **{name}:** {value}" for name, value in details)] ) - lines.extend( - [ - "", - "#### Remediation", - "", - _text(finding["remediation"], "No canonical remediation was recorded."), - ] - ) - if remediation_tests: - lines.extend(["", "Tests:", *_bullets(remediation_tests, "No tests recorded.")]) - if preventive_controls: - lines.extend(["", "Preventive controls:", *_bullets(preventive_controls, "None recorded.")]) + lines.extend(_remediation_section(finding)) return lines @@ -769,8 +808,12 @@ def _linked_finding_section(number: int, finding: dict[str, Any], report_path: s f"| CWE | {_cell(cwes)} |", f"| Affected lines | {_cell(_locations(finding))} |", ] - for heading in ("Summary", "Validation", "Dataflow", "Reachability", "Severity", "Remediation"): + for heading in ("Summary", "Validation", "Dataflow", "Reachability", "Severity"): lines.extend(["", f"#### {heading}", "", f"See the {link}."]) + if finding.get("provenance", {}).get("sourceFindings"): + lines.extend(_remediation_section(finding)) + else: + lines.extend(["", "#### Remediation", "", f"See the {link}."]) return lines @@ -986,8 +1029,8 @@ def build_report_markdown( "No vulnerability conclusion can be drawn." if no_source_review else ( - "No findings were validated before the scan reached its cost limit. " - "Review the deferred candidates in Open Questions And Follow Up." + "No findings are included in this partial report. " + "Review the unresolved work in Open Questions And Follow Up." if budget_exhausted else "No reportable findings survived the canonical discovery, validation, " "and reportability gates." @@ -1007,6 +1050,22 @@ def build_report_markdown( f"[Open the structural hardening portfolio]({hardening_portfolio_path})", ] ) + reviews = coverage.get("reviews", []) + if reviews: + lines.extend( + [ + "", + "## Source Review Coverage", + "", + "| Review | Attempt | Coverage |", + "| --- | --- | --- |", + ] + ) + for review in reviews: + if isinstance(review, dict): + lines.append( + f"| {_cell(review.get('workerId'))} | {_cell(str(review.get('attempt', 'unknown')))} | {_cell(review.get('completeness'))} |" + ) surfaces = coverage.get("surfaces", []) if surfaces: lines.extend( @@ -1044,6 +1103,7 @@ def build_report_markdown( questions.extend( { "question": item.get("reason", "Deferred review requires follow-up."), + "provenance": item.get("provenance", {}), "followUpPrompt": " ".join( ( f"Review deferred unit {item.get('id', 'unknown')} and close its stated proof gap.", @@ -1065,6 +1125,9 @@ def build_report_markdown( if not isinstance(question, dict): continue lines.append(f"- {_text(question.get('question'), 'Unspecified open question.')}") + source = _coverage_source(question) + if source: + lines.append(f" - {_text(source, '')}.") prompt = _text(question.get("followUpPrompt"), "") if prompt: lines.append(f" - Follow-up prompt: {prompt}") diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.py index dc538fd41..d21447ef3 100644 --- a/plugins/codex-security/scripts/workbench_cli.py +++ b/plugins/codex-security/scripts/workbench_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import json import sys from pathlib import Path @@ -30,7 +31,7 @@ def add_user_context(parser: argparse.ArgumentParser, *, required: bool = False) context.add_argument("--user-context-stdin", action="store_true") -def parse_args(description: str) -> argparse.Namespace: +def parse_args(description: str, *, execution_settings: bool = False) -> argparse.Namespace: parser = argparse.ArgumentParser(description=description) subparsers = parser.add_subparsers(dest="command", required=True) @@ -392,7 +393,16 @@ def parse_args(description: str) -> argparse.Namespace: parser.error("pass exactly one user-context transport") index = arguments.index("--user-context-stdin") arguments[index] = "--user-context=" + sys.stdin.buffer.read().decode("utf-8") - return parser.parse_args(arguments) + args = parser.parse_args(arguments) + if execution_settings: + args.require_execution_settings = True + if args.command == "begin-deep-scan": + # Private MCP creation input shares stdin with the user's unchanged context. + payload = json.load(sys.stdin) + args.execution_settings = payload["executionSettings"] + args.user_context = payload.get("userContext") + args.user_context_stdin = False + return args def non_negative_int(value: str) -> int: diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py index 161756ee0..38be4a228 100644 --- a/plugins/codex-security/scripts/workbench_db.py +++ b/plugins/codex-security/scripts/workbench_db.py @@ -54,9 +54,7 @@ _prepare_scan_finalization, _write_prepared_scan_finalization, finalize_scan, - finding_candidate_id, open_scan_local_file_descriptor, - write_scan_local_bytes, ) from finding_preview import bounded_finding_details from workbench import handoff @@ -1155,18 +1153,22 @@ def complete_budget_exhausted_scan( connection: sqlite3.Connection, args: argparse.Namespace ) -> dict[str, Any]: scan_id = require_uuid(args.scan_id, "scan-id") - cost_json = parse_scan_cost(args.cost_json) + cost_json = parse_scan_cost(args.cost_json, allow_lower_bound=True) if cost_json is None: raise SystemExit("Budget-exhausted scan completion requires the measured scan cost.") with scan_completion_lock(scan_id): scan = require_scan(connection, scan_id) if scan["status"] != "running" or scan["mode"] != "deep" or scan["recipe_json"] is None: raise SystemExit("Only a running CLI Deep Scan can complete after its cost limit.") + handoff.require_current_continuation( + scan, None, error_message="Scan completion is owned by another continuation." + ) recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json) if not isinstance(recipe, dict) or recipe.get("mode") != "deep": raise SystemExit("Budget-exhausted scan completion requires a Deep Scan launch recipe.") cost = json.loads(cost_json) - measured = cost.get("cost", cost) + lower_bound = set(cost) == {"lowerBound"} + measured = cost["lowerBound"] if lower_bound else cost.get("cost", cost) limit = recipe.get("maxCostUsd") if ( not isinstance(limit, (int, float)) @@ -1175,11 +1177,13 @@ def complete_budget_exhausted_scan( or measured.get("estimatedUsd", 0) <= limit ): raise SystemExit("Deep Scan has not exceeded its configured cost limit.") - run = connection.execute( - "SELECT status, terminal_reason, manifest_path FROM deep_scan_runs WHERE scan_id = ?", - (scan_id,), - ).fetchone() - if ( + run = deep_scan.find_supported_deep_scan_run(connection, scan_id) + before_selection = ( + run is not None + and run["status"] == "running" + and run["workflow_version"] == "deep-security-scan/v2" + ) + if not before_selection and ( run is None or run["status"] != "succeeded" or run["terminal_reason"] not in {"saturated", "capped"} @@ -1192,16 +1196,44 @@ def complete_budget_exhausted_scan( scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) candidates = ( [] - if run["manifest_path"] == str(scan_dir / "scan-manifest.json") + if before_selection or run["manifest_path"] == str(scan_dir / "scan-manifest.json") else budget_exhausted_candidates(scan, scan_dir) ) warning = optional_text(args.message, maximum=2400) if warning is None: + retained = ( + "saved work was retained" + if before_selection + else "completed discovery was preserved" + ) warning = ( f"Deep Scan reached its cost limit after an estimated " - f"${measured['estimatedUsd']:.6g}; completed discovery was preserved." + f"${measured['estimatedUsd']:.6g}; {retained}." + ) + if before_selection: + deep_scan.prepare_budget_exhausted_deep_scan( + connection, + scan, + scan_dir, + warning, + lambda accepted, unmerged: budget_exhausted_draft( + scan, + scan_dir, + [], + warning, + before_selection=True, + accepted_result=accepted, + unmerged_workers=unmerged, + ), + ) + else: + budget_exhausted_draft( + scan, + scan_dir, + candidates, + warning, + unmerged_workers=deep_scan.budget_unmerged_workers(connection, scan_id), ) - budget_exhausted_draft(scan, scan_dir, candidates, warning) warnings = json.loads(scan["completion_warnings_json"]) if warning not in warnings: connection.execute( @@ -1209,7 +1241,8 @@ def complete_budget_exhausted_scan( (json.dumps([*warnings, warning]), scan_id), ) connection.commit() - return complete_scan_locked(connection, scan_id, None, cost_json) + # A priced subtotal proves the stop but is not the saved total estimate. + return complete_scan_locked(connection, scan_id, None, None if lower_bound else cost_json) def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[str, Any]]: @@ -1289,148 +1322,50 @@ def budget_exhausted_draft( scan_dir: Path, candidates: list[dict[str, Any]], warning: str, + *, + before_selection: bool = False, + accepted_result: dict[str, Any] | None = None, + unmerged_workers: list[sqlite3.Row] = (), ) -> None: - documents: dict[str, dict[str, Any]] = {} - for name in ("scan-manifest.json", "findings.json", "coverage.json"): - path = artifact_path(scan_dir, name, required=False) - if path is not None: - documents[name] = read_json_object(path) - if documents and len(documents) != 3: - raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") - - if documents: - manifest = documents["scan-manifest.json"] - findings = documents["findings.json"] - coverage = documents["coverage.json"] - if not isinstance(manifest.get("scan"), dict) or not isinstance( - findings.get("findings"), list - ): - raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.") - for key in ("surfaces", "explicitExclusions", "deferred"): - if not isinstance(coverage.get(key), list): - raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.") - if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"): - raise SystemExit("Budget-exhausted scan cannot replace an already sealed scan draft.") - else: - contract = scan_contract(scan) - target_contract = contract["target"] - target: dict[str, Any] = { - "kind": target_contract["allowedKinds"][0], - "targetId": target_contract["targetId"], - "displayName": target_contract["displayName"], - } - if scan["target_revision"] != "unversioned": - target["revision"] = scan["target_revision"] - if "requiredSnapshotDigest" in target_contract: - target["snapshotDigest"] = target_contract["requiredSnapshotDigest"] - manifest = { - "scan": { - "target": target, - "scope": {"limitations": [warning], "validationMode": "incomplete"}, - } - } - findings = {"findings": []} - coverage = { - "completeness": "partial", - "inventoryStrategy": ( - "scoped_path" if expected_coverage_mode(scan) == "scoped_path" else "repository" - ), - "surfaces": [], - "explicitExclusions": [], - "deferred": [], - } + saved_results.budget_exhausted_draft( + _WORKBENCH_DB_CONTEXT, + scan, + scan_dir, + candidates, + warning, + scan_contract(scan), + before_selection=before_selection, + accepted_result=accepted_result, + unmerged_workers=unmerged_workers, + ) - findings_by_candidate = { - candidate_id - for finding in findings["findings"] - if isinstance(finding, dict) - and isinstance(candidate_id := finding_candidate_id(finding), str) - } - existing_deferred = { - item.get("candidateId", item.get("id")) - for item in coverage["deferred"] - if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str) - } - existing_surfaces = { - item.get("id") - for item in coverage["surfaces"] - if isinstance(item, dict) and isinstance(item.get("id"), str) + +def completed_scan_context( + connection: sqlite3.Connection, scan_id: str, coverage: dict[str, Any] +) -> dict[str, Any]: + context = scan_context(connection, scan_id) + context["coverageSummary"] = { + "completeness": coverage["completeness"], + "surfaceCount": len(coverage["surfaces"]), + "deferredCount": len(coverage.get("deferred", [])), + "explicitExclusionCount": len(coverage.get("explicitExclusions", [])), } - for candidate in candidates: - candidate_id = candidate["candidate_id"] - if candidate_id in findings_by_candidate or candidate_id in existing_deferred: - continue - paths = list(dict.fromkeys(location["path"] for location in candidate["locations"])) - surface_id = f"candidate-{candidate_id}" - validation = candidate.get("validation") - validation = validation.get("disposition") if isinstance(validation, dict) else None - attack = candidate.get("attack_path") - attack = attack.get("decision") if isinstance(attack, dict) else None - disposition = ( - "needs_follow_up" - if validation == "deferred" or attack == "deferred" - else "not_applicable" - if validation == "not_applicable" - else "rejected" - if validation == "suppressed" or attack == "ignore" - else "needs_follow_up" - ) - if surface_id not in existing_surfaces: - coverage["surfaces"].append( - { - "id": surface_id, - "label": candidate["summary"], - "disposition": disposition, - "notes": candidate["evidence"], - "receiptRefs": [], - } - ) - existing_surfaces.add(surface_id) - if disposition != "needs_follow_up": - continue - coverage["deferred"].append( - { - "id": candidate_id, - "candidateId": candidate_id, - "reason": ( - "Validation was deferred because the scan reached its cost limit: " - f"{candidate['summary']}. Evidence: {candidate['evidence']}" - ), - "paths": paths, - "surfaceIds": [surface_id], - } - ) - if not any( - isinstance(item, dict) - and isinstance(reason := item.get("reason"), str) - and ( - reason == "Validation was deferred because the scan reached its cost limit." - or reason.startswith( - "Validation was deferred because the scan reached its cost limit: " - ) - ) - for item in coverage["deferred"] - ): - coverage["deferred"].append( - { - "id": "scan-cost-limit", - "reason": "Validation was deferred because the scan reached its cost limit.", - } - ) - coverage["completeness"] = "partial" - for name, payload in ( - ("findings.json", findings), - ("coverage.json", coverage), - ("scan-manifest.json", manifest), - ): - try: - write_scan_local_bytes( - scan_dir, - name, - (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(), - ) - except (ContractError, OSError, TypeError, ValueError) as exc: - raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + return context + + +def validate_completed_scan_artifacts(scan: sqlite3.Row) -> None: + scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) + require_recorded_manifest_digest(scan, scan_dir) + verify_manifest_binding(scan, read_json_object(scan_dir / ARTIFACTS["manifest"])) + try: + manifest = _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=expected_coverage_mode(scan), + )[2] + except ContractError as exc: + raise SystemExit(str(exc)) from exc + verify_manifest_binding(scan, manifest) + published_manifest_digest(scan_dir, manifest) def complete_scan_locked( @@ -1444,11 +1379,12 @@ def complete_scan_locked( ) -> dict[str, Any]: scan = require_scan(connection, scan_id) if scan["status"] == "complete": + deep_scan.find_supported_deep_scan_run(connection, scan_id) scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"])) require_recorded_manifest_digest(scan, scan_dir) verify_manifest_binding(scan, read_json_object(scan_dir / ARTIFACTS["manifest"])) try: - manifest, _, _ = finalize_scan( + manifest, _, coverage = finalize_scan( scan_dir, expected_coverage_mode=expected_coverage_mode(scan), ) @@ -1459,7 +1395,7 @@ def complete_scan_locked( pin_legacy_manifest_digest(connection, scan["id"], manifest_digest) if cost_json is not None and scan["recipe_json"] is not None: scan_usage.reconcile_completed_scan_cost(connection, scan, cost_json) - return scan_context(connection, scan["id"]) + return completed_scan_context(connection, scan["id"], coverage) if scan["status"] != "running": raise SystemExit("Only a running scan can be completed.") handoff.require_current_continuation( @@ -1545,10 +1481,12 @@ def add_warning() -> None: ) add_warning() wrote = True - manifest, findings, _ = _write_prepared_scan_finalization(prepared) + manifest, findings, coverage = _write_prepared_scan_finalization(prepared) except ContractError as exc: - if wrote or ( + # Replay a validated Deep aggregate after an output write fails. + if (wrote and scan["mode"] != "deep") or ( scan["mode"] == "deep" + and not wrote and not already_sealed and not isinstance(exc, RecoverableContractError) ): @@ -1592,7 +1530,7 @@ def add_warning() -> None: scan = require_scan(connection, scan["id"]) if scan["status"] == "complete": connection.commit() - return scan_context(connection, scan["id"]) + return completed_scan_context(connection, scan["id"], coverage) if scan["status"] != "running": raise SystemExit("Only a running scan can be completed.") deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan) @@ -1643,7 +1581,7 @@ def add_warning() -> None: except BaseException: connection.rollback() raise - context = scan_context(connection, scan["id"]) + context = completed_scan_context(connection, scan["id"], coverage) context["targetWarnings"] = target_warnings return context @@ -2839,8 +2777,7 @@ def scan_result( **scan_usage.stored_scan_cost_fields(scan["cost_json"]), "contract": scan_contract(scan), "continuationThreadId": scan["continuation_thread_id"], - "threadIds": scan_usage._scan_root_thread_ids(connection, scan, None), - "executionThreadIds": scan_usage._scan_execution_thread_ids(connection, scan), + **scan_usage.scan_execution_fields(connection, scan), "failureMessage": scan["failure_message"], "findings": [ finding_result(connection, scan, row, related=relations.get(row["id"], [])) @@ -3402,10 +3339,10 @@ def read_json_object(path: Path) -> dict[str, Any]: ) -def main() -> None: +def main(*, select_finalization: bool = False, with_execution_settings: bool = False) -> None: # Workbench callers send UTF-8 even when Windows uses a legacy code page. sys.stdin.reconfigure(encoding="utf-8") - args = parse_args(__doc__) + args = parse_args(__doc__, execution_settings=with_execution_settings) deep_scan.configure( deep_scan.DeepScanDependencies( now=now, @@ -3466,6 +3403,13 @@ def main() -> None: result = start_headless_standard_scan(connection, args) elif args.command == "begin-deep-scan": result = deep_scan.begin_deep_scan(connection, args) + if ( + result["deepScan"]["status"] == "succeeded" + and result["deepScan"]["finalizationInput"] is not None + ): + scan = require_scan(connection, result["deepScan"]["scanId"]) + if scan["status"] == "complete": + validate_completed_scan_artifacts(scan) elif args.command == "get-deep-scan": result = deep_scan.get_deep_scan(connection, args) elif args.command == "claim-deep-scan-coordinator": @@ -3477,7 +3421,7 @@ def main() -> None: elif args.command == "commit-deep-scan-dedup": result = deep_scan.commit_deep_scan_dedup(connection, args) elif args.command == "finish-deep-scan": - result = deep_scan.finish_deep_scan(connection, args) + result = deep_scan.finish_deep_scan(connection, args, select_finalization) elif args.command == "fail-deep-scan": result = deep_scan.fail_deep_scan(connection, args) elif args.command == "record-deep-scan-publication-failure": diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index a32b96513..fa80f3757 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -104,6 +104,35 @@ def _children(scan_dir: Path, relative: str) -> list[str]: return sorted(child.name for child in cursor.iterdir()) +def _saved_workers(connection: Any, scan_id: str) -> list[dict[str, Any]]: + rows = connection.execute( + "SELECT worker.*, attempt.accepted_result_path FROM deep_scan_workers AS worker " + "LEFT JOIN deep_scan_attempts AS attempt " + "ON attempt.worker_id = worker.id AND attempt.attempt = worker.attempt " + "WHERE worker.scan_id = ? ORDER BY worker.created_at, worker.id", + (scan_id,), + ).fetchall() + return [ + { + **dict(row), + "result_manifest_path": row["accepted_result_path"] or row["result_manifest_path"], + } + for row in rows + ] + + +def _accepted_source_digests(connection: Any, scan_id: str) -> dict[str, str]: + return { + row["accepted_result_path"]: row["accepted_result_sha256"] + for row in connection.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE scan_id = ? AND accepted_result_path IS NOT NULL " + "AND accepted_result_sha256 IS NOT NULL", + (scan_id,), + ) + } + + def _latest_successful_reducer(workers: list[Any]) -> Any | None: return max( ( @@ -161,18 +190,81 @@ def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, def _read_saved_result( - scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None + scan_dir: Path, + relative: str, + scan_id: str, + *, + kind: str | None = None, + accepted_source_digests: dict[str, str] | None = None, ) -> tuple[dict[str, Any], str]: - draft = _read_scan_local_json(scan_dir, relative, "Saved scan checkpoint") + draft, contents = _read_scan_local_json_bytes(scan_dir, relative, "Saved scan checkpoint") + expected = (accepted_source_digests or {}).get(str(scan_dir / relative)) + if expected is not None and hashlib.sha256(contents).hexdigest() != expected: + raise ContractError("checkpoint changed after acceptance") if draft.get("scanId") != scan_id: raise ContractError("checkpoint belongs to a different scan") - if not isinstance(draft.get("findings"), list) or not isinstance( - draft.get("coverage", {} if kind == "dedup" else None), dict - ): + coverage = ( + draft.get("sourceCoverage", draft.get("coverage", {})) + if kind == "dedup" + else draft.get("coverage") + ) + if not isinstance(draft.get("findings"), list) or not isinstance(coverage, dict): raise ContractError("checkpoint has no semantic findings or coverage") return draft, _digest(draft) +def _worker_checkpoint_head( + scan_dir: Path, + directory: str, + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> str | None: + relative = f"{directory}/checkpoint-head.json" + try: + (scan_dir / relative).lstat() + except FileNotFoundError: + return None + head = _read_scan_local_json(scan_dir, relative, "Saved worker checkpoint head") + name = head.get("checkpoint") + if not isinstance(name, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", name): + raise ContractError("Saved worker checkpoint head is invalid.") + checkpoint = f"{directory}/checkpoints/{name}" + # A committed head precedes replacement of result.json. Do not fall back to + # that older result if the selected checkpoint cannot be read. + _read_saved_result( + scan_dir, checkpoint, scan_id, accepted_source_digests=accepted_source_digests + ) + return checkpoint + + +def _worker_checkpoint_heads( + scan_dir: Path, + workers: list[Any], + scan_id: str, + accepted_source_digests: dict[str, str] | None = None, +) -> dict[str, str]: + heads: dict[str, str] = {} + for worker in workers: + if worker["kind"] != "discovery": + continue + try: + output = Path(worker["artifact_dir"]).relative_to(scan_dir) + except (TypeError, ValueError): + continue + attempts = (output.parent if output.name == "output" else output) / "attempts" + directories = [output] + [ + attempts / name + for name in _children(scan_dir, attempts.as_posix()) + if re.fullmatch(r"attempt-\d+", name) + ] + for directory in directories: + relative = directory.as_posix() + head = _worker_checkpoint_head(scan_dir, relative, scan_id, accepted_source_digests) + if head is not None: + heads[relative] = head + return heads + + def _read_saved_parent_result( scan_dir: Path, scan_id: str ) -> tuple[dict[str, Any], dict[str, Any]]: @@ -215,18 +307,21 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool: try: scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False) - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) paths = dict(_saved_result_paths(scan_dir, workers)) frozen_sources = scan["retained_source_digests_json"] def has_saved_source() -> bool: for path in paths: try: - _read_saved_result(scan_dir, path, scan["id"], kind=paths[path]) + _read_saved_result( + scan_dir, + path, + scan["id"], + kind=paths[path], + accepted_source_digests=accepted_digests, + ) return True except (ContractError, OSError, ValueError): continue @@ -254,11 +349,19 @@ def has_saved_source() -> bool: published_sources = _source_digests( manifest_scan.get("preservedSources", {}), "Published scan" ) + if _worker_checkpoint_heads( + scan_dir, workers, scan["id"], accepted_digests + ) != manifest_scan.get("preservedCheckpointHeads", {}): + return True current_sources = dict(published_sources) for path in paths: try: _, current_sources[path] = _read_saved_result( - scan_dir, path, scan["id"], kind=paths[path] + scan_dir, + path, + scan["id"], + kind=paths[path], + accepted_source_digests=accepted_digests, ) except (ContractError, OSError, ValueError): continue @@ -267,7 +370,9 @@ def has_saved_source() -> bool: return False -def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]: +def _recovery_source_digests( + db: Any, connection: Any, scan: Any +) -> tuple[dict[str, str], bool, dict[str, str]]: scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) frozen_sources: dict[str, str] | None = None include_parent = True @@ -305,16 +410,20 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ else: include_parent = True - workers = connection.execute( - "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path " - "FROM deep_scan_workers WHERE scan_id = ?", - (scan["id"],), - ).fetchall() + workers = _saved_workers(connection, scan["id"]) + accepted_digests = _accepted_source_digests(connection, scan["id"]) + checkpoint_heads = _worker_checkpoint_heads(scan_dir, workers, scan["id"], accepted_digests) paths = dict(_saved_result_paths(scan_dir, workers)) recovery_sources = dict(frozen_sources or {}) for relative, expected_digest in recovery_sources.items(): try: - _, digest = _read_saved_result(scan_dir, relative, scan["id"], kind=paths.get(relative)) + _, digest = _read_saved_result( + scan_dir, + relative, + scan["id"], + kind=paths.get(relative), + accepted_source_digests=accepted_digests, + ) except (ContractError, OSError, ValueError) as exc: raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc if digest != expected_digest: @@ -323,11 +432,15 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[ for relative in paths.keys() - recovery_sources.keys(): try: _, recovery_sources[relative] = _read_saved_result( - scan_dir, relative, scan["id"], kind=paths[relative] + scan_dir, + relative, + scan["id"], + kind=paths[relative], + accepted_source_digests=accepted_digests, ) except (ContractError, OSError, ValueError): continue - return recovery_sources, include_parent + return recovery_sources, include_parent, checkpoint_heads def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool: @@ -433,6 +546,30 @@ def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> N finding["identity"] = {"anchor": anchor} +def _ensure_finding_identities(findings: list[dict[str, Any]]) -> None: + def key(finding: dict[str, Any]) -> bytes: + identity = finding["identity"] + return _encoded( + [finding.get("ruleId"), identity.get("anchor"), identity.get("instance", "")] + ) + + # Reserve authored identities even when they follow a generated sibling. + reserved = { + key(finding) + for finding in findings + if isinstance(finding, dict) and isinstance(finding.get("identity"), dict) + } + for finding in findings: + if not isinstance(finding, dict) or "identity" in finding: + continue + _ensure_finding_identity(finding) + suffix = 2 + while key(finding) in reserved: + finding["identity"]["instance"] = f"saved-{suffix}" + suffix += 1 + reserved.add(key(finding)) + + def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]: """Yield canonical and historical findings without trusting candidate IDs.""" pending = [finding] @@ -469,10 +606,16 @@ def merge_saved_results( stopped: bool, reason: str, frozen_source_digests: dict[str, str] | None = None, + checkpoint_heads: dict[str, str] | None = None, allow_frozen_legacy_parent: bool = False, + accepted_source_digests: dict[str, str] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None: """Read only bound parent/worker files; return an unsealed loss-preserving union.""" initial_warnings = set(warnings) + if checkpoint_heads is None: + checkpoint_heads = _worker_checkpoint_heads( + scan_dir, workers, scan_id, accepted_source_digests + ) parent: dict[str, Any] | None = None parent_manifest: dict[str, Any] | None = None if frozen_source_digests is None or allow_frozen_legacy_parent: @@ -519,6 +662,7 @@ def merge_saved_results( reducer_paths.add(latest_reducer) except ValueError: warnings.append("Skipped a reducer result outside the scan directory.") + accepted_reducer = latest_reducer def checkpoints(directory: str, worker_id: str | None) -> None: for name in _children(scan_dir, directory): @@ -560,22 +704,40 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: continue if worker["kind"] != "discovery": continue + head = checkpoint_heads.get(output) + if head is not None: + paths[head] = worker["id"] + current_results.add(head) paths[f"{output}/result.json"] = worker["id"] - current_results.add(f"{output}/result.json") + if head is None: + current_results.add(f"{output}/result.json") checkpoints(f"{output}/checkpoints", worker["id"]) attempts = ( Path(output).parent if Path(output).name == "output" else Path(output) ) / "attempts" - for name in _children(scan_dir, attempts.as_posix()): - if re.fullmatch(r"attempt-\d+", name): - archived = (attempts / name).as_posix() - paths[f"{archived}/result.json"] = worker["id"] - checkpoints(f"{archived}/checkpoints", worker["id"]) + archived_attempts = sorted( + ( + name + for name in _children(scan_dir, attempts.as_posix()) + if re.fullmatch(r"attempt-\d+", name) + ), + key=lambda name: int(name.removeprefix("attempt-")), + reverse=True, + ) + for name in archived_attempts: + archived = (attempts / name).as_posix() + archived_head = checkpoint_heads.get(archived) + if archived_head is not None: + paths[archived_head] = worker["id"] + current_results.add(archived_head) + paths[f"{archived}/result.json"] = worker["id"] + checkpoints(f"{archived}/checkpoints", worker["id"]) if worker["result_manifest_path"]: try: current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[current_path] = worker["id"] - current_results.add(current_path) + if head is None: + current_results.add(current_path) except ValueError: warnings.append("Skipped a worker result outside the scan directory.") @@ -592,14 +754,23 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: for relative, worker_id in paths.items(): try: draft, digest = _read_saved_result( - scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None + scan_dir, + relative, + scan_id, + kind="dedup" if relative in reducer_paths else None, + accepted_source_digests=accepted_source_digests, ) if frozen_source_digests is not None and frozen_source_digests[relative] != digest: raise ContractError("checkpoint changed after the scan stopped") source_digests[relative] = digest - # Recovery expects coverage, but reducer results only contain findings - # and context. Add an empty value after hashing the original result. - sources.append((relative, {"coverage": {}, **draft}, worker_id)) + # The host supplies reducer coverage separately from model output. + # Preserve the digest of the original accepted document. + if relative in reducer_paths: + draft = { + **draft, + "coverage": draft.get("sourceCoverage", draft.get("coverage", {})), + } + sources.append((relative, draft, worker_id)) except (ContractError, OSError, ValueError) as exc: if (scan_dir / relative).exists(): warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}") @@ -608,6 +779,15 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") drafts_by_path = {relative: draft for relative, draft, _ in sources} + if "sourceCoverage" not in drafts_by_path.get(accepted_reducer, {}): + accepted_reducer = None + accepted_coverage = drafts_by_path.get(accepted_reducer, {}).get("coverage", {}) + reviewed_attempts = { + (review.get("workerId"), review.get("attempt")) + for review in accepted_coverage.get("reviews", []) + if isinstance(review, dict) + } + workers_by_id = {worker["id"]: worker for worker in workers} latest_reducer_key = ( (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0)) if reducer is not None and latest_reducer in drafts_by_path @@ -637,6 +817,7 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: and parent_manifest["scan"].get("sealedAt") and parent_manifest["scan"].get("status") == binding["status"] and parent_manifest["scan"].get("preservedSources") == source_digests + and parent_manifest["scan"].get("preservedCheckpointHeads", {}) == checkpoint_heads and all(warning in initial_warnings for warning in warnings) ): return None @@ -668,6 +849,7 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: for key in ("sealedAt", "artifacts"): manifest["scan"].pop(key, None) manifest["scan"]["preservedSources"] = source_digests + manifest["scan"]["preservedCheckpointHeads"] = checkpoint_heads coverage = ( copy.deepcopy(parent["coverage"]) if parent and parent["coverage"] @@ -722,8 +904,19 @@ def valid_finding(value: Any) -> bool: all_sources = ([("parent", parent, None)] if parent else []) + sources current_drafts = ([(None, parent)] if parent else []) + [ - (worker_id, draft) for relative, draft, worker_id in sources if relative in current_results + (worker_id, draft) + for relative, draft, worker_id in sources + if relative in current_results or relative == accepted_reducer ] + + def coverage_candidate(owner: str | None, item: dict[str, Any]) -> tuple[str | None, Any]: + provenance = item.get("provenance") + if owner is None and isinstance(provenance, dict): + return provenance.get("workerId"), provenance.get( + "candidateId", item.get("candidateId") + ) + return owner, item.get("candidateId") + resolved: dict[tuple[str | None, str], str] = {} for owner, draft in current_drafts: for finding in draft["findings"]: @@ -741,7 +934,7 @@ def valid_finding(value: Any) -> bool: and isinstance(item.get("candidateId"), str) and item.get("disposition") in {"reported", "rejected", "not_applicable"} ): - resolved.setdefault((owner, item["candidateId"]), item["disposition"]) + resolved.setdefault(coverage_candidate(owner, item), item["disposition"]) # Only the current parent may claim that another worker finding was absorbed. # A superseded checkpoint must not suppress a newer independent result. for draft in [parent] if parent else []: @@ -955,11 +1148,18 @@ def valid_finding(value: Any) -> bool: continue finding_positions[key] = len(findings) findings.append(finding) - if superseded: + worker = workers_by_id.get(worker_id) + reviewed = ( + worker is not None + and worker["status"] == "succeeded" + and worker["merge_state"] == "merged" + and (worker_id, worker["attempt"]) in reviewed_attempts + ) + if reviewed or (superseded and relative != accepted_reducer): continue - for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"): items = draft["coverage"].get(field, []) - if not isinstance(items, list): + if not isinstance(items, list) or (field == "reviews" and not items): continue output = coverage.setdefault(field, []) if not isinstance(output, list): @@ -990,7 +1190,7 @@ def valid_finding(value: Any) -> bool: history.append(copy.deepcopy(finding)) if ( isinstance(item, dict) - and (worker_id, item.get("candidateId")) in resolved + and coverage_candidate(worker_id, item) in resolved and (field == "deferred" or item.get("disposition") == "needs_follow_up") ): continue @@ -1048,6 +1248,306 @@ def valid_finding(value: Any) -> bool: return manifest, {"findings": findings}, coverage +def budget_exhausted_draft( + db: WorkbenchDbContext, + scan: sqlite3.Row, + scan_dir: Path, + candidates: list[dict[str, Any]], + warning: str, + contract: dict[str, Any], + *, + before_selection: bool = False, + accepted_result: dict[str, Any] | None = None, + unmerged_workers: list[sqlite3.Row] = (), +) -> None: + documents: dict[str, dict[str, Any]] = {} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + path = db.artifact_path(scan_dir, name, required=False) + if path is not None: + documents[name] = db.read_json_object(path) + interrupted_documents = {} + if documents and len(documents) != 3: + if not before_selection or set(documents) not in ( + {"findings.json"}, + {"findings.json", "coverage.json"}, + ): + raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") + interrupted_documents = documents + documents = {} + + if documents: + manifest = documents["scan-manifest.json"] + findings = documents["findings.json"] + coverage = documents["coverage.json"] + if not isinstance(manifest.get("scan"), dict) or not isinstance( + findings.get("findings"), list + ): + raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.") + for key in ("surfaces", "explicitExclusions", "deferred"): + if not isinstance(coverage.get(key), list): + raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.") + if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"): + if before_selection: + raise SystemExit( + "Budget-exhausted scan cannot replace an already sealed scan draft." + ) + # The seal can reach disk before parent completion commits. Validate + # it without changing bytes; the existing finalizer commits replay. + try: + _prepare_scan_finalization( + scan_dir, + expected_coverage_mode=db.expected_coverage_mode(scan), + completion_binding=db.workbench_completion_binding( + scan, scan["started_at"], manifest + ), + ) + except ContractError as exc: + raise SystemExit(str(exc)) from exc + return + else: + target_contract = contract["target"] + target: dict[str, Any] = { + "kind": target_contract["allowedKinds"][0], + "targetId": target_contract["targetId"], + "displayName": target_contract["displayName"], + } + if scan["target_revision"] != "unversioned": + target["revision"] = scan["target_revision"] + if "requiredSnapshotDigest" in target_contract: + target["snapshotDigest"] = target_contract["requiredSnapshotDigest"] + manifest = { + "scan": { + "target": target, + "scope": {"limitations": [warning], "validationMode": "incomplete"}, + } + } + findings = {"findings": []} + coverage = { + "completeness": "partial", + "inventoryStrategy": ( + "scoped_path" if db.expected_coverage_mode(scan) == "scoped_path" else "repository" + ), + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + } + + if before_selection: + # Unmerged discoveries remain in their accepted artifacts. Only the + # committed reducer may contribute findings to this partial result. + findings = { + "findings": copy.deepcopy(accepted_result["findings"]) if accepted_result else [] + } + _ensure_finding_identities(findings["findings"]) + if accepted_result is not None: + coverage = accepted_result["sourceCoverage"] + if "threatModel" in accepted_result: + manifest["scan"]["threatModel"] = accepted_result["threatModel"] + if "scope" in accepted_result: + manifest["scan"]["scope"].update(copy.deepcopy(accepted_result["scope"])) + limitations = manifest["scan"]["scope"].setdefault("limitations", []) + if warning not in limitations: + limitations.append(warning) + + for worker in unmerged_workers: + retain_unmerged_budget_coverage(scan, scan_dir, coverage, worker) + + findings_by_candidate = { + candidate_id + for finding in findings["findings"] + if isinstance(finding, dict) + and isinstance(candidate_id := finding_candidate_id(finding), str) + } + existing_deferred = { + item.get("candidateId", item.get("id")) + for item in coverage["deferred"] + if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str) + } + existing_surfaces = { + item.get("id") + for item in coverage["surfaces"] + if isinstance(item, dict) and isinstance(item.get("id"), str) + } + for candidate in candidates: + candidate_id = candidate["candidate_id"] + if candidate_id in findings_by_candidate or candidate_id in existing_deferred: + continue + paths = list(dict.fromkeys(location["path"] for location in candidate["locations"])) + surface_id = f"candidate-{candidate_id}" + validation = candidate.get("validation") + validation = validation.get("disposition") if isinstance(validation, dict) else None + attack = candidate.get("attack_path") + attack = attack.get("decision") if isinstance(attack, dict) else None + disposition = ( + "needs_follow_up" + if validation == "deferred" or attack == "deferred" + else "not_applicable" + if validation == "not_applicable" + else "rejected" + if validation == "suppressed" or attack == "ignore" + else "needs_follow_up" + ) + if surface_id not in existing_surfaces: + coverage["surfaces"].append( + { + "id": surface_id, + "label": candidate["summary"], + "disposition": disposition, + "notes": candidate["evidence"], + "receiptRefs": [], + } + ) + existing_surfaces.add(surface_id) + if disposition != "needs_follow_up": + continue + coverage["deferred"].append( + { + "id": candidate_id, + "candidateId": candidate_id, + "reason": ( + "Validation was deferred because the scan reached its cost limit: " + f"{candidate['summary']}. Evidence: {candidate['evidence']}" + ), + "paths": paths, + "surfaceIds": [surface_id], + } + ) + if not any( + isinstance(item, dict) + and isinstance(reason := item.get("reason"), str) + and ( + reason == "Validation was deferred because the scan reached its cost limit." + or reason.startswith( + "Validation was deferred because the scan reached its cost limit: " + ) + ) + for item in coverage["deferred"] + ): + coverage["deferred"].append( + { + "id": "scan-cost-limit", + "reason": "Validation was deferred because the scan reached its cost limit.", + } + ) + coverage["completeness"] = "partial" + outputs = ( + ("findings.json", findings), + ("coverage.json", coverage), + ("scan-manifest.json", manifest), + ) + # Process death can leave a prefix of these derived writes before SQLite + # commits. Recover only bytes whose documents match the accepted inputs. + if any( + interrupted_documents[name] != payload + for name, payload in outputs + if name in interrupted_documents + ): + raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.") + for name, payload in outputs: + try: + write_scan_local_bytes( + scan_dir, + name, + (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(), + ) + except (ContractError, OSError, TypeError, ValueError) as exc: + raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc + + +def retain_unmerged_budget_coverage( + scan: sqlite3.Row, scan_dir: Path, coverage: dict[str, Any], worker: sqlite3.Row +) -> None: + """Keep each unmerged review's obligations; its findings remain evidence only.""" + relative = Path(worker["accepted_result_path"]).relative_to(scan_dir).as_posix() + draft, _ = _read_saved_result( + scan_dir, + relative, + scan["id"], + accepted_source_digests={worker["accepted_result_path"]: worker["accepted_result_sha256"]}, + ) + source = draft["coverage"] + provenance = {"workerId": worker["id"], "attempt": worker["attempt"]} + prefix = f"{worker['id']}-attempt-{worker['attempt']}" + artifact_prefix = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix() + surfaces = { + item.get("id"): f"{prefix}-surface-{index + 1}" + for index, item in enumerate(source.get("surfaces", [])) + } + + def retain(field: str, item: dict[str, Any]) -> None: + # A committed budget draft can be replayed before the scan is sealed. + # These IDs and provenance identify the same immutable accepted review. + items = coverage.setdefault(field, []) + if "id" in item: + matches = [] + for index, existing in enumerate(items): + existing_provenance = ( + existing.get("provenance") if isinstance(existing, dict) else None + ) + if ( + isinstance(existing_provenance, dict) + and existing.get("id") == item["id"] + and all( + existing_provenance.get(key) == value for key, value in provenance.items() + ) + ): + matches.append(index) + if matches: + # Refresh older projections from the same accepted bytes, even + # if an interrupted writer saved more than one copy. + items[matches[0]] = item + for index in reversed(matches[1:]): + del items[index] + return + if item not in items: + items.append(item) + + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + for index, original in enumerate(source.get(field, [])): + item = copy.deepcopy(original if isinstance(original, dict) else {"question": original}) + source_provenance = item.get("provenance") + if not isinstance(source_provenance, dict): + source_provenance = {} + # Keep source descriptions; the accepted owner supplies identity. + for key in ("workerId", "attempt", "sourceId", "candidateId"): + source_provenance.pop(key, None) + item["provenance"] = { + **source_provenance, + **provenance, + **({"sourceId": item["id"]} if "id" in item else {}), + **({"candidateId": item["candidateId"]} if "candidateId" in item else {}), + } + item["id"] = f"{prefix}-{field}-{index + 1}" + if field == "surfaces": + item["id"] = f"{prefix}-surface-{index + 1}" + item["receiptRefs"] = [ + f"{artifact_prefix}/{ref}" for ref in item.get("receiptRefs", []) + ] + if field == "deferred" and "candidateId" in item: + item["candidateId"] = f"{prefix}-candidate-{index + 1}" + if "surfaceIds" in item: + item["surfaceIds"] = [surfaces.get(value, value) for value in item["surfaceIds"]] + retain(field, item) + retain("reviews", {**provenance, "completeness": source["completeness"]}) + for index, limitation in enumerate(draft.get("scope", {}).get("limitations", [])): + retain( + "deferred", + { + "id": f"{prefix}-scope-{index + 1}", + "reason": limitation, + "provenance": provenance, + }, + ) + retain( + "deferred", + { + "id": f"{prefix}-unmerged", + "provenance": provenance, + "reason": "This accepted discovery was not merged before the scan reached its cost limit.", + }, + ) + + def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]: if scan["seal_manifest_digest"] is None: raise SystemExit("Only sealed scans can be compared.") @@ -1100,6 +1600,7 @@ def preserve_scan_results_locked( scan_id: str, *, recovery_source_digests: dict[str, str] | None = None, + recovery_checkpoint_heads: dict[str, str] | None = None, include_parent_with_recovery: bool = False, ) -> bool: """Publish or verify retained terminal results through the workbench host.""" @@ -1107,6 +1608,7 @@ def preserve_scan_results_locked( if scan["status"] != "failed": return False frozen_source_digests: dict[str, str] | None = None + checkpoint_heads = recovery_checkpoint_heads raw_frozen_sources = scan["retained_source_digests_json"] if recovery_source_digests is not None: frozen_source_digests = recovery_source_digests @@ -1114,10 +1616,13 @@ def preserve_scan_results_locked( frozen_source_digests = _source_digests( json.loads(raw_frozen_sources), "Saved stopped-scan" ) + checkpoint_heads = json.loads(scan["retained_checkpoint_heads_json"] or "{}") scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) deep_run = connection.execute( - "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) ).fetchone() + if deep_run is not None: + db.deep_scan.require_supported_deep_scan(deep_run) outcome = ( "canceled" if scan["canceled_at"] @@ -1167,11 +1672,15 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No db.index_findings(connection, scan_id, findings, scan["completed_at"]) connection.execute( "UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, " + "retained_checkpoint_heads_json = ?, " "completion_warnings_json = ?, " "updated_at = ? WHERE id = ? AND status = 'failed'", ( digest, json.dumps(retained_sources, sort_keys=True), + json.dumps( + manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True + ), json.dumps(list(dict.fromkeys(warnings))), timestamp, scan_id, @@ -1198,6 +1707,7 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No db.verify_manifest_binding(scan, existing) if existing_scan.get("status") == outcome: existing_sources = existing_scan.get("preservedSources") + existing_heads = existing_scan.get("preservedCheckpointHeads", {}) if frozen_source_digests is None: if not isinstance(existing_sources, dict) or not all( isinstance(relative, str) and isinstance(digest, str) @@ -1205,7 +1715,8 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No ): raise ContractError("Stopped scan source digests could not be frozen.") frozen_source_digests = existing_sources - if existing_sources == frozen_source_digests: + checkpoint_heads = existing_heads + if existing_sources == frozen_source_digests and existing_heads == checkpoint_heads: if ( raw_frozen_sources is not None and scan["seal_manifest_digest"] is not None @@ -1221,10 +1732,7 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No scan_dir, scan_id, binding, - connection.execute( - "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id", - (scan_id,), - ).fetchall(), + _saved_workers(connection, scan_id), warnings, stopped=True, reason=( @@ -1232,6 +1740,8 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No f"{scan['failure_message'] or ''}" ).strip(), frozen_source_digests=frozen_source_digests, + checkpoint_heads=checkpoint_heads, + accepted_source_digests=_accepted_source_digests(connection, scan_id), allow_frozen_legacy_parent=( include_parent_with_recovery or ( @@ -1263,9 +1773,16 @@ def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> No frozen_source_digests = retained_sources with connection: connection.execute( - "UPDATE scans SET retained_source_digests_json = ? " + "UPDATE scans SET retained_source_digests_json = ?, " + "retained_checkpoint_heads_json = ? " "WHERE id = ? AND retained_source_digests_json IS NULL", - (json.dumps(retained_sources, sort_keys=True), scan_id), + ( + json.dumps(retained_sources, sort_keys=True), + json.dumps( + documents[0]["scan"].get("preservedCheckpointHeads", {}), sort_keys=True + ), + scan_id, + ), ) prepared = _prepare_scan_finalization( scan_dir, @@ -1293,12 +1810,15 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]: raise SystemExit("Only a stopped scan can recover terminal results.") if scan["canceled_at"] is not None: raise SystemExit("Canceled scans cannot recover terminal results.") - recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan) + recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests( + db, connection, scan + ) if not preserve_scan_results_locked( db, connection, scan_id, recovery_source_digests=recovery_source_digests, + recovery_checkpoint_heads=checkpoint_heads, include_parent_with_recovery=include_parent, ): raise SystemExit("No saved stopped-scan results were available to recover.") @@ -1383,6 +1903,48 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]: return {"scanId": scan_id, "path": str(scan_dir / output)} +def _read_staged_scan_draft(scan_dir: Path, draft_path: str) -> dict[str, Any]: + try: + relative = Path(draft_path).relative_to(scan_dir).as_posix() + except ValueError as exc: + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") from exc + if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative): + raise SystemExit("Scan draft must be inside the registered scan drafts directory.") + return _read_scan_local_json(scan_dir, relative, "Staged scan draft") + + +def _require_current_deep_publication( + db: Any, connection: Any, scan_id: str, draft: dict[str, Any] +) -> None: + publication = draft.get("deepScanPublication") + run = db.deep_scan.require_deep_scan_run(connection, scan_id) + db.deep_scan.require_current_coordinator( + run, + argparse.Namespace( + coordinator_generation=publication.get("coordinatorGeneration") if publication else None + ), + ) + selection = db.deep_scan.deep_scan_finalization_input(run) + if selection is not None: + if publication is None: + raise SystemExit("Deep Scan publication requires its committed selection.") + scan = db.require_scan(connection, scan_id) + selected_result = ( + str(Path(scan["scan_dir"]) / selection["resultPath"]) + if selection["resultPath"] is not None + else None + ) + else: + # Generation-one runs predate host publication metadata. Keep their existing + # draft path; adopted coordinators must carry their generation and selection. + if publication is None: + return + reducer = _latest_successful_reducer(_saved_workers(connection, scan_id)) + selected_result = reducer["result_manifest_path"] if reducer is not None else None + if publication["resultPath"] != selected_result: + raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.") + + def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: scan_id = db.require_uuid(args.scan_id, "scan-id") with db.scan_completion_lock(scan_id): @@ -1395,6 +1957,10 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: "The scan stopped; its saved checkpoint was retained without replacing sealed results." ) scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"])) + draft = None + if scan["mode"] == "deep": + draft = _read_staged_scan_draft(scan_dir, args.draft_path) + _require_current_deep_publication(db, connection, scan_id, draft) if args.checkpoint_path is not None: try: checkpoint_relative = Path(args.checkpoint_path).relative_to(scan_dir).as_posix() @@ -1424,15 +1990,8 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]: raise SystemExit( "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again." ) - try: - relative = Path(args.draft_path).relative_to(scan_dir).as_posix() - except ValueError as exc: - raise SystemExit( - "Scan draft must be inside the registered scan drafts directory." - ) from exc - if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative): - raise SystemExit("Scan draft must be inside the registered scan drafts directory.") - draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft") + if draft is None: + draft = _read_staged_scan_draft(scan_dir, args.draft_path) manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"] binding = db.workbench_completion_binding(scan, db.now()) # Validate on copies: saved canonical documents remain ordinary unsealed drafts. diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py index fc145bc04..e814adda9 100644 --- a/plugins/codex-security/scripts/workbench_scan_history.py +++ b/plugins/codex-security/scripts/workbench_scan_history.py @@ -75,10 +75,16 @@ def cli_scan_resume( ): raise SystemExit("Resume requires the original owning CLI session.") run = connection.execute( - "SELECT status, cancel_requested FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + "SELECT status, cancel_requested, finalization_input_json FROM deep_scan_runs " + "WHERE scan_id = ?", + (scan["id"],), ).fetchone() if run is not None and ( - run["status"] not in {"running", "succeeded"} or run["cancel_requested"] + run["status"] not in {"running", "succeeded"} + or ( + run["cancel_requested"] + and not (run["status"] == "succeeded" and run["finalization_input_json"] is not None) + ) ): raise SystemExit("This Deep Scan has stopped and cannot resume.") try: diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py index 648924d92..b00787117 100644 --- a/plugins/codex-security/scripts/workbench_scan_usage.py +++ b/plugins/codex-security/scripts/workbench_scan_usage.py @@ -9,6 +9,7 @@ import sqlite3 import sys import uuid +from collections import deque from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -90,12 +91,54 @@ def collect_scan_usage( ) -> dict[str, Any]: """Count only complete, attributable rollout events inside this scan's window.""" - roots = _scan_root_thread_ids(connection, scan, thread_id) + attribution = scan_execution_attribution(connection, scan) + if attribution and attribution.get("legacy"): + attribution = None + roots = ( + list( + dict.fromkeys( + [ + *( + [attribution["owner"]["threadId"]] + if attribution["owner"].get("threadId") + else [] + ), + *attribution["executionThreadIds"], + ] + ) + ) + if attribution + else _scan_root_thread_ids(connection, scan, thread_id) + ) if not roots: return _unavailable_usage("scan_thread_unavailable") - state_database = _codex_state_database() - if state_database is None: + warnings: set[str] = set() + current_database = _codex_state_database() + worker_codex_home = None + if scan["mode"] == "deep": + # Deep orchestration imports the owner-capture helper from this module; + # its settings reader is available once completion starts. + from deep_scan_workbench import read_deep_scan_execution_settings + + try: + settings = read_deep_scan_execution_settings(Path(scan["scan_dir"])) + worker_codex_home = Path(settings["codexHome"]) + except SystemExit: + # Legacy scans may have no recorded home. Keep usage best effort. + pass + groups = [(current_database, roots)] + worker_roots: set[str] = set() + if worker_codex_home is not None: + worker_roots = set( + _scan_root_thread_ids(connection, scan, None, include_owner_threads=False) + ) + # Workers retain their Codex home, but inherit an explicit current + # SQLite home. Their earlier and resumed sessions can be in either index. + worker_database = _codex_state_database(worker_codex_home) + if worker_database != current_database: + groups.append((worker_database, [root for root in roots if root in worker_roots])) + if not any(database is not None for database, _ in groups) and not worker_roots: return _unavailable_usage("codex_state_unavailable") started_at = _timestamp(scan["started_at"]) @@ -103,24 +146,75 @@ def collect_scan_usage( if started_at is None: return _unavailable_usage("scan_window_unavailable") - warnings: set[str] = set() - try: - sessions, missing_thread_ids = _discover_rollout_sessions( - state_database, - roots, - warnings, + sessions: dict[str, list[RolloutSession]] = {} + missing_thread_ids: set[str] = set() + seen_thread_ids: set[str] = set() + for state_database, group_roots in groups: + if not group_roots: + continue + try: + if state_database is None: + raise FileNotFoundError("Codex state is unavailable") + discovered, missing = _discover_rollout_sessions( + state_database, + group_roots, + warnings, + descendant_roots=set(attribution["executionThreadIds"]) if attribution else None, + ) + except (OSError, sqlite3.Error, ValueError): + warnings.add("codex_state_unavailable") + missing_thread_ids.update(group_roots) + continue + missing_thread_ids.update(missing) + for session in discovered: + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + if worker_codex_home is not None and worker_roots: + # An external SQLite location can change on recovery. Native rollouts + # still live in the recorded worker home; they retain their lineage. + for session in _discover_recorded_worker_sessions(worker_codex_home, worker_roots): + copies = sessions.setdefault(session.thread_id, []) + if session not in copies: + copies.append(session) + seen_thread_ids.add(session.thread_id) + + # Absence from one known index is not missing usage when another has it. + missing_thread_ids.difference_update(seen_thread_ids) + if not missing_thread_ids: + warnings.difference_update( + {"scan_root_unavailable", "codex_state_unavailable", "rollout_unavailable"} ) - except (OSError, sqlite3.Error, ValueError): - return _unavailable_usage("codex_state_unavailable") if not sessions: - return _unavailable_usage("scan_thread_unavailable", warnings=warnings) + return _unavailable_usage( + "codex_state_unavailable" + if "codex_state_unavailable" in warnings + else "scan_thread_unavailable", + warnings=warnings, + ) total = _empty_token_usage() observed_thread_count = 0 accepted_thread_ids: set[str] = set() excluded_thread_ids: set[str] = set() - for session in sessions: + model_usage: dict[str | None, dict[str, int]] = {} + for copies in sessions.values(): + session = copies[0] + owner_turn_id = None + if ( + attribution + and session.thread_id not in attribution["executionThreadIds"] + and session.parent_thread_id is None + ): + owner = attribution["owner"] + if session.thread_id != owner.get("threadId") or not owner.get("turnId"): + missing_thread_ids.add(session.thread_id) + warnings.add("scan_owner_turn_unavailable") + continue + owner_turn_id = owner["turnId"] if session.parent_thread_id in excluded_thread_ids: excluded_thread_ids.add(session.thread_id) continue @@ -132,10 +226,12 @@ def collect_scan_usage( warnings.add("thread_lineage_incomplete") continue try: - session_usage, session_warnings = _read_rollout_usage( - session, + session_usage, session_warnings = _read_rollout_copies_usage( + copies, started_at=started_at, completed_at=stopped_at, + owner_turn_id=owner_turn_id, + model_usage=model_usage, ) except (OSError, UnicodeError, ValueError): missing_thread_ids.add(session.thread_id) @@ -151,6 +247,9 @@ def collect_scan_usage( missing_thread_ids.add(session.thread_id) continue accepted_thread_ids.add(session.thread_id) + if "token_usage_unavailable" in session_warnings: + missing_thread_ids.add(session.thread_id) + continue observed_thread_count += 1 _add_token_usage(total, session_usage) @@ -167,9 +266,57 @@ def collect_scan_usage( result["missingThreadCount"] = len(missing_thread_ids) if warnings: result["warnings"] = sorted(warnings) + if attribution or any(model is not None for model in model_usage): + result["modelUsage"] = [{"model": model, **usage} for model, usage in model_usage.items()] return result +def _read_rollout_copies_usage( + copies: list[RolloutSession], + *, + started_at: datetime, + completed_at: datetime | None, + owner_turn_id: str | None, + model_usage: dict[str | None, dict[str, int]], +) -> tuple[dict[str, int], set[str]]: + readings = [] + for session in copies: + local_models: dict[str | None, dict[str, int]] = {} + try: + usage, warnings = _read_rollout_usage( + session, + started_at=started_at, + completed_at=completed_at, + owner_turn_id=owner_turn_id, + model_usage=local_models, + ) + except (OSError, UnicodeError, ValueError): + continue + readings.append((usage, warnings, local_models)) + if not readings: + raise ValueError("No readable rollout copy.") + attributable = [ + reading + for reading in readings + if not reading[1].intersection( + { + "thread_identity_mismatch", + "thread_ownership_unavailable", + "thread_outside_scan_window", + "token_usage_unavailable", + } + ) + ] + # Restored indexes can reference a prefix and its complete continuation. + # Keep totals and model attribution from the same copy, counting it once. + usage, warnings, selected_models = max( + attributable or readings, key=lambda reading: reading[0]["totalTokens"] + ) + for model, tokens in selected_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), tokens) + return usage, warnings + + def _scan_root_thread_ids( connection: sqlite3.Connection, scan: sqlite3.Row, @@ -194,12 +341,13 @@ def _scan_root_thread_ids( row["sdk_thread_id"] for row in connection.execute( """ - SELECT DISTINCT sdk_thread_id - FROM deep_scan_workers + SELECT sdk_thread_id FROM deep_scan_attempt_sessions WHERE scan_id = ? + UNION + SELECT sdk_thread_id FROM deep_scan_workers WHERE scan_id = ? AND sdk_thread_id IS NOT NULL ORDER BY sdk_thread_id """, - (scan["id"],), + (scan["id"], scan["id"]), ) ) roots: list[str] = [] @@ -221,15 +369,105 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row ) -def _codex_state_database() -> Path | None: - configured_database = os.environ.get("CODEX_STATE_DB", "").strip() +def capture_scan_usage_owner(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + """Bind the active native turn once; joining a scan does not bind later conversation work.""" + roots = _scan_root_thread_ids(connection, scan, None) + owner = roots[0] if roots else None + result = { + "threadId": owner, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + database = _codex_state_database() + if owner is None or database is None: + return result + try: + sessions, _ = _discover_rollout_sessions(database, [owner], set(), descendant_roots=set()) + if not sessions: + return result + with sessions[0].path.open("rb") as source: + for line in source: + if not line.endswith(b"\n"): + continue + event = json.loads(line) + payload = event.get("payload") + if not isinstance(payload, dict): + continue + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + turn_id = payload.get("turn_id") + if isinstance(turn_id, str): + result["turnId"] = turn_id + elif event.get("type") == "event_msg" and payload.get("type") == "task_complete": + result["turnId"] = None + except (OSError, ValueError, sqlite3.Error): + # Accounting availability must not prevent a scan from starting. + pass + return result + + +def scan_execution_attribution( + connection: sqlite3.Connection, scan: sqlite3.Row +) -> dict[str, Any] | None: + if scan["mode"] != "deep": + return None + run = connection.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],) + ).fetchone() + if run is None: + return None + owner_json = run["usage_owner_json"] if "usage_owner_json" in run.keys() else None + legacy = False + if owner_json is None: + legacy = ( + connection.execute( + "SELECT 1 FROM deep_scan_attempts WHERE scan_id = ? LIMIT 1", (scan["id"],) + ).fetchone() + is None + ) + roots = _scan_root_thread_ids(connection, scan, None) + owner = { + "threadId": roots[0] if roots else None, + "turnId": None, + "startedAt": scan["started_at"], + "dedicated": scan["recipe_json"] is not None, + } + else: + owner = json.loads(owner_json) + executions = _scan_execution_thread_ids(connection, scan) + if owner.get("dedicated") and owner.get("threadId") not in executions: + executions.append(owner["threadId"]) + return { + "formatVersion": 1, + **({"legacy": True} if legacy else {}), + "executionThreadIds": executions, + "owner": owner, + "startedAt": scan["started_at"], + "completedAt": scan["completed_at"], + } + + +def scan_execution_fields(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + return { + "threadIds": _scan_root_thread_ids(connection, scan, None), + "executionThreadIds": _scan_execution_thread_ids(connection, scan), + "executionAttribution": scan_execution_attribution(connection, scan), + } + + +def _codex_state_database(worker_codex_home: Path | None = None) -> Path | None: + configured_home = os.environ.get("CODEX_HOME", "").strip() + current_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex" + codex_home = worker_codex_home if worker_codex_home is not None else current_home + same_home = worker_codex_home is None or codex_home.resolve() == current_home.resolve() + configured_database = os.environ.get("CODEX_STATE_DB", "").strip() if same_home else "" if configured_database: path = Path(configured_database).expanduser() return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None - configured_home = os.environ.get("CODEX_HOME", "").strip() - codex_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex" - configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip() + configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip() if same_home else "" search_roots = [ *([Path(configured_sqlite_home).expanduser()] if configured_sqlite_home else []), codex_home, @@ -260,6 +498,8 @@ def _discover_rollout_sessions( state_database: Path, roots: list[str], warnings: set[str], + *, + descendant_roots: set[str] | None = None, ) -> tuple[list[RolloutSession], set[str]]: database = sqlite3.connect( state_database.as_uri() + "?mode=ro", @@ -295,6 +535,8 @@ def _discover_rollout_sessions( continue sessions.append(RolloutSession(root, None, path)) seen_thread_ids.add(root) + if descendant_roots is not None and root not in descendant_roots: + continue descendants = database.execute( """ WITH RECURSIVE descendants( @@ -357,6 +599,50 @@ def _discover_rollout_sessions( database.close() +def _discover_recorded_worker_sessions(codex_home: Path, roots: set[str]) -> list[RolloutSession]: + recorded: dict[str, list[RolloutSession]] = {} + children: dict[str, set[str]] = {} + for candidate in sorted((codex_home / "sessions").rglob("*.jsonl")): + path = _rollout_path(str(candidate)) + if path is None: + continue + try: + with path.open("rb") as stream: + metadata = json.loads(stream.readline()) + except (OSError, UnicodeError, ValueError): + continue + if not isinstance(metadata, dict) or metadata.get("type") != "session_meta": + continue + payload = metadata.get("payload") + if not isinstance(payload, dict): + continue + thread_id = payload.get("id") or payload.get("session_id") + if not isinstance(thread_id, str): + continue + parent_id = _session_parent_thread_id(payload) + recorded.setdefault(thread_id, []).append(RolloutSession(thread_id, parent_id, path)) + if parent_id is not None: + children.setdefault(parent_id, set()).add(thread_id) + + sessions: list[RolloutSession] = [] + included = set(roots) + pending = deque(sorted(roots)) + while pending: + thread_id = pending.popleft() + for session in recorded.get(thread_id, []): + sessions.append( + RolloutSession( + thread_id, + None if thread_id in roots else session.parent_thread_id, + session.path, + ) + ) + for child_id in sorted(children.get(thread_id, set()) - included): + included.add(child_id) + pending.append(child_id) + return sessions + + def _require_state_columns( connection: sqlite3.Connection, table: str, @@ -399,11 +685,22 @@ def _read_rollout_usage( *, started_at: datetime, completed_at: datetime | None, + owner_turn_id: str | None = None, + model_usage: dict[str | None, dict[str, int]] | None = None, ) -> tuple[dict[str, int], set[str]]: total = _empty_token_usage() + counter_total = _empty_token_usage() warnings: set[str] = set() previous = _empty_token_usage() boundary_reached = False + usage_observed = False + current_turn_id: str | None = None + current_model: str | None = None + response_ids: set[str] = set() + response_usage_observed = False + response_tokens = 0 + expected_response_tokens = 0 + local_models: dict[str | None, dict[str, int]] = {} with session.path.open("rb") as source: for line_number, raw_line in enumerate(source, start=1): @@ -423,6 +720,10 @@ def _read_rollout_usage( warnings.add("rollout_record_invalid") continue payload = event.get("payload") + if event.get("type") in {"session_meta", "turn_context"} and isinstance(payload, dict): + model = payload.get("model") + if isinstance(model, str) and model: + current_model = model if line_number == 1: if event.get("type") != "session_meta" or not isinstance(payload, dict): warnings.add("thread_identity_mismatch") @@ -445,6 +746,10 @@ def _read_rollout_usage( if not isinstance(payload, dict): continue + if event.get("type") == "turn_context" or ( + event.get("type") == "event_msg" and payload.get("type") == "task_started" + ): + current_turn_id = payload.get("turn_id") if not boundary_reached: if _is_owned_task_start(session.thread_id, event, payload): task_started_at = _timestamp(event.get("timestamp")) @@ -462,28 +767,97 @@ def _read_rollout_usage( if inherited_usage is not None: previous = inherited_usage continue + if event.get("type") == "token_usage_record": + response_id = payload.get("response_id") + usage = _token_snapshot({"info": {"total_token_usage": payload.get("usage")}}) + if ( + not isinstance(response_id, str) + or usage is None + or payload.get("thread_id", session.thread_id) != session.thread_id + or response_id in response_ids + ): + continue + response_ids.add(response_id) + cumulative = _token_snapshot( + {"info": {"total_token_usage": payload.get("thread_token_usage")}} + ) + if cumulative is not None: + expected_response_tokens = max( + expected_response_tokens, cumulative["totalTokens"] + ) + if not response_usage_observed: + response_usage_observed = True + total = _empty_token_usage() + local_models = {} + response_tokens += usage["totalTokens"] + timestamp = _timestamp(event.get("timestamp")) + if timestamp is None: + warnings.add("token_record_invalid") + continue + if timestamp < started_at or ( + completed_at is not None and timestamp > completed_at + ): + continue + if ( + owner_turn_id is not None + and payload.get("turn_id", current_turn_id) != owner_turn_id + ): + continue + usage_observed = True + model = payload.get("model", current_model) + if not isinstance(model, str): + model = None + _add_token_usage(total, usage) + _add_token_usage(local_models.setdefault(model, _empty_token_usage()), usage) + continue if event.get("type") != "event_msg" or payload.get("type") != "token_count": continue + # Native rate-limit updates can carry no token usage. + if "info" in payload and payload["info"] is None: + continue timestamp = _timestamp(event.get("timestamp")) snapshot = _token_snapshot(payload) if timestamp is None or snapshot is None: warnings.add("token_record_invalid") continue - delta = { - key: value - previous[key] if value >= previous[key] else value - for key, value in snapshot.items() - } + if snapshot["totalTokens"] < previous["totalTokens"]: + warnings.add("token_counter_regressed") + continue + delta = {key: max(0, value - previous[key]) for key, value in snapshot.items()} previous = snapshot if timestamp < started_at: continue if completed_at is not None and timestamp > completed_at: continue + if owner_turn_id is not None and current_turn_id != owner_turn_id: + continue + usage_observed = True + if not response_usage_observed: + local_models.setdefault(current_model, _empty_token_usage()) if delta["totalTokens"] <= 0: continue - _add_token_usage(total, delta) + _add_token_usage(counter_total, delta) + if not response_usage_observed: + _add_token_usage(total, delta) + _add_token_usage( + local_models.setdefault(current_model, _empty_token_usage()), delta + ) + if counter_total["totalTokens"] > total["totalTokens"]: + remainder = {key: max(0, value - total[key]) for key, value in counter_total.items()} + total = dict(counter_total) + _add_token_usage(local_models.setdefault(None, _empty_token_usage()), remainder) + if response_usage_observed: + warnings.discard("token_counter_regressed") + if expected_response_tokens > response_tokens: + warnings.add("token_receipts_incomplete") + if model_usage is not None: + for model, usage in local_models.items(): + _add_token_usage(model_usage.setdefault(model, _empty_token_usage()), usage) if not boundary_reached: warnings.add("thread_ownership_unavailable") + elif not usage_observed: + warnings.add("token_usage_unavailable") return total, warnings diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py index 66baa2b4f..9c69dcbfc 100644 --- a/plugins/codex-security/scripts/workbench_schema.py +++ b/plugins/codex-security/scripts/workbench_schema.py @@ -867,6 +867,85 @@ ); """, ), + ( + 44, + "preserve original deep scan discovery context", + """ + ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT; + UPDATE deep_scan_runs + SET discovery_user_context = ( + SELECT user_context FROM scans WHERE scans.id = deep_scan_runs.scan_id + ) + WHERE workflow_version IN ( + 'deep-security-scan/v1', 'deep-scan-mcp/v1' + ); + """, + ), + ( + 45, + "retain deep scan attempts and exact merge inputs", + """ + CREATE TABLE deep_scan_attempts ( + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + worker_id TEXT NOT NULL REFERENCES deep_scan_workers(id) ON DELETE CASCADE, + attempt INTEGER NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + end_reason TEXT, + error_message TEXT, + accepted_result_path TEXT, + accepted_result_sha256 TEXT, + receipt_json TEXT, + PRIMARY KEY (worker_id, attempt) + ); + + CREATE TABLE deep_scan_attempt_sessions ( + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + worker_id TEXT NOT NULL, + attempt INTEGER NOT NULL, + sdk_thread_id TEXT NOT NULL, + observed_at TEXT NOT NULL, + PRIMARY KEY (worker_id, attempt, sdk_thread_id), + FOREIGN KEY (worker_id, attempt) + REFERENCES deep_scan_attempts(worker_id, attempt) ON DELETE CASCADE + ); + + CREATE TABLE deep_scan_merge_claims ( + worker_id TEXT PRIMARY KEY REFERENCES deep_scan_workers(id) ON DELETE CASCADE, + scan_id TEXT NOT NULL REFERENCES deep_scan_runs(scan_id) ON DELETE CASCADE, + previous_worker_id TEXT, + previous_result_path TEXT, + previous_result_sha256 TEXT, + receipt_json TEXT + ); + + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_path TEXT; + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN result_manifest_sha256 TEXT; + ALTER TABLE deep_scan_dedup_inputs ADD COLUMN attempt INTEGER; + """, + ), + ( + 46, + "persist selected deep scan finalization input", + """ + ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT; + """, + ), + ( + 47, + "freeze stopped scan checkpoint selections", + """ + ALTER TABLE scans ADD COLUMN retained_checkpoint_heads_json TEXT; + """, + ), + ( + 48, + "bind original deep scan parent usage turn", + """ + ALTER TABLE deep_scan_runs ADD COLUMN usage_owner_json TEXT; + """, + ), ) diff --git a/plugins/codex-security/scripts/workbench_validation.py b/plugins/codex-security/scripts/workbench_validation.py index a65aff1e7..a511fcc51 100644 --- a/plugins/codex-security/scripts/workbench_validation.py +++ b/plugins/codex-security/scripts/workbench_validation.py @@ -137,6 +137,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: "threadCount", "missingThreadCount", "warnings", + "modelUsage", *SCAN_USAGE_TOKEN_KEYS, } if thread_count == 0 or not set(usage).issubset(allowed_keys): @@ -144,6 +145,19 @@ def _valid_measured_scan_usage(usage: object) -> bool: counts = {key: usage.get(key) for key in SCAN_USAGE_TOKEN_KEYS} if not _valid_scan_token_counts(counts): return False + if "modelUsage" in usage: + parts = usage["modelUsage"] + if not isinstance(parts, list) or not parts: + return False + for part in parts: + if not isinstance(part, dict) or set(part) != {"model", *SCAN_USAGE_TOKEN_KEYS}: + return False + if part["model"] is not None and not isinstance(part["model"], str): + return False + if not _valid_scan_token_counts({key: part[key] for key in SCAN_USAGE_TOKEN_KEYS}): + return False + if any(sum(part[key] for part in parts) != counts[key] for key in SCAN_USAGE_TOKEN_KEYS): + return False missing = usage.get("missingThreadCount", 0) if type(missing) is not int or missing < 0: return False @@ -154,7 +168,7 @@ def _valid_measured_scan_usage(usage: object) -> bool: return True -def parse_scan_cost(value: str | None) -> str | None: +def parse_scan_cost(value: str | None, *, allow_lower_bound: bool = False) -> str | None: if value is None: return None if len(value.encode("utf-8")) > 8192: @@ -163,7 +177,10 @@ def parse_scan_cost(value: str | None) -> str | None: cost = json.loads(value, parse_constant=reject_nonstandard_json_number) except (TypeError, UnicodeError, ValueError) as exc: raise SystemExit("Scan cost must be a valid JSON object.") from exc - if isinstance(cost, dict) and "usage" in cost: + if allow_lower_bound and isinstance(cost, dict) and set(cost) == {"lowerBound"}: + if not _valid_legacy_scan_cost(cost["lowerBound"]): + raise SystemExit("Scan cost lower bound must be a valid measured cost.") + elif isinstance(cost, dict) and "usage" in cost: if ( not set(cost).issubset({"usage", "cost"}) or not _valid_measured_scan_usage(cost["usage"]) diff --git a/plugins/codex-security/tests/test_accepted_publication_references.py b/plugins/codex-security/tests/test_accepted_publication_references.py new file mode 100644 index 000000000..e7d12d871 --- /dev/null +++ b/plugins/codex-security/tests/test_accepted_publication_references.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def accept_reducer(connection, scan): + result = add_worker(connection, scan) + worker_id = result.parent.name + coverage = { + **scan.coverage, + "completeness": "partial", + "deferred": [{"id": "accepted-follow-up", "reason": "Accepted unresolved review."}], + "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}], + } + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "sourceCoverage": coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + result.write_bytes(contents) + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + result.unlink() + with connection: + connection.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE id = ?", + (worker_id,), + ) + connection.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + return result, accepted, coverage + + +@pytest.mark.parametrize("selected", [True, False], ids=["accepted", "replaceable-output"]) +def test_legacy_publication_compares_registered_accepted_reference( + workbench_api, workbench_db, publication_scan, selected +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + staged = stage_publication( + scan, generation=3, result_path=accepted if selected else result, title="Accepted aggregate" + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if selected: + workbench_api["write_scan_draft"](workbench_db, staged) + else: + with pytest.raises(SystemExit, match="aggregate"): + workbench_api["write_scan_draft"](workbench_db, staged) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + + +def test_stopped_recovery_uses_accepted_bytes_after_replaceable_output_disappears( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + contents = accepted.read_bytes() + + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["findingCount"] == 1 + published = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert published["reviews"] == coverage["reviews"] + assert coverage["deferred"][0] in published["deferred"] + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_budget_coverage_provenance.py b/plugins/codex-security/tests/test_budget_coverage_provenance.py new file mode 100644 index 000000000..4ce460726 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_coverage_provenance.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes +from test_workbench_db import BUDGET_COST + + +@pytest.mark.parametrize("explicit_ids", [True, False], ids=["named-surfaces", "omitted-ids"]) +@pytest.mark.parametrize("replay", ["fresh", "legacy", "duplicated"]) +def test_cost_completion_retains_independent_unmerged_surfaces_and_receipts( + workbench_api, + workbench_db, + publication_scan, + explicit_ids, + replay, + tmp_path, + monkeypatch, +): + scan = publication_scan() + worker_id = add_worker(workbench_db, scan).parent.name + output = scan.scan_dir / "artifacts/deep_discovery/workers/discovery-0001/output" + output.mkdir(parents=True) + descriptions = { + "source": "source-review", + "worker": "focused-reviewer", + "task": "Check the filesystem boundary.", + } + source_provenance = { + **descriptions, + "workerId": "worker-local-label", + "attempt": 99, + "sourceId": "worker-local-source", + "candidateId": "worker-local-candidate", + } + surfaces = [ + { + "label": "Filesystem boundary", + "disposition": "needs_follow_up", + "notes": "Filesystem race remains untested.", + "reason": "The caller's filesystem policy is unknown.", + }, + { + "label": "Template boundary", + "disposition": "needs_follow_up", + "notes": "Template caller policy remains untested.", + }, + { + "label": "Configuration boundary", + "disposition": "rejected", + "reason": "The input comes only from trusted application configuration.", + }, + ] + receipts = {} + for index, surface in enumerate(surfaces, start=1): + surface["provenance"] = source_provenance + if explicit_ids: + surface["id"] = f"source-surface-{index}" + relative = f"artifacts/review-{index}.md" + surface["receiptRefs"] = [relative] + receipt = output / relative + receipt.parent.mkdir(exist_ok=True) + receipts[receipt] = f"Independent review receipt {index}.\n".encode() + receipt.write_bytes(receipts[receipt]) + deferred = { + "reason": "Both independent validation tasks remain unfinished.", + "provenance": source_provenance, + } + if explicit_ids: + deferred["surfaceIds"] = [surface["id"] for surface in surfaces] + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": { + **scan.coverage, + "completeness": "partial", + "surfaces": surfaces, + "deferred": [deferred], + "explicitExclusions": [ + { + "pattern": "vendor/", + "reason": "External dependencies were excluded.", + "provenance": source_provenance, + } + ], + "openQuestions": [ + { + "question": "Which deployment controls apply?", + "provenance": source_provenance, + } + ], + }, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = output / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + # The mutable result may advance after acceptance. Publication must read the + # accepted checkpoint, while receipt paths remain relative to its owner. + result = output / "result.json" + result.write_text("A later, unaccepted result.") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ?, " + "prompt_path = ?, merge_state = 'buffered' WHERE id = ?", + (str(output), str(result), str(output.parent / "prompt.md"), worker_id), + ) + workbench_db.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL, max_discovery_runs = 100" + ) + counter_query = ( + "SELECT discovery_runs_dispatched, completion_sequence, consecutive_no_new " + "FROM deep_scan_runs" + ) + counters = tuple(workbench_db.execute(counter_query).fetchone()) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + + budget = workbench_api["complete_budget_exhausted_scan"] + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Scan reached its original cost limit.", + ) + if replay != "fresh": + saved = budget.__globals__["saved_results"] + retain = saved.retain_unmerged_budget_coverage + + def legacy_projection(*args): + retain(*args) + # Older writers kept host identity but omitted source descriptions. + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + items = args[2].get(field, []) + for item in list(items): + if item.get("provenance", {}).get("workerId") == worker_id: + if replay == "duplicated" and any( + key in item["provenance"] for key in descriptions + ): + # A second interrupted writer appended the described + # projection beside the old item before sealing. + items.append(copy.deepcopy(item)) + for key in descriptions: + item["provenance"].pop(key, None) + + def interrupt_before_seal(*args): + raise RuntimeError("Publication interrupted after the budget draft committed.") + + database = tmp_path / "interrupted-budget.sqlite3" + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_db.backup(connection) + with monkeypatch.context() as patch: + patch.setattr(saved, "retain_unmerged_budget_coverage", legacy_projection) + patch.setitem(budget.__globals__, "complete_scan_locked", interrupt_before_seal) + with pytest.raises(RuntimeError, match="budget draft committed"): + budget(connection, args) + # Reconnect to the committed old draft and run the real public completion. + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + assert ( + connection.execute("SELECT status FROM deep_scan_runs").fetchone()[0] == "succeeded" + ) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + connection.execute("SELECT seal_manifest_digest FROM scans").fetchone()[0] is None + ) + budget(connection, args) + connection.backup(workbench_db) + else: + budget(workbench_db, args) + + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + actual = coverage["surfaces"] + assert [(item["label"], item.get("notes"), item.get("reason")) for item in actual] == [ + (item["label"], item.get("notes"), item.get("reason")) for item in surfaces + ] + assert len({item["id"] for item in actual}) == len(surfaces) + for original, retained in zip(surfaces, actual, strict=True): + assert retained["provenance"] == { + **descriptions, + "workerId": worker_id, + "attempt": 1, + **({"sourceId": original["id"]} if explicit_ids else {}), + } + assert retained["receiptRefs"] == [ + (output / ref).relative_to(scan.scan_dir).as_posix() for ref in original["receiptRefs"] + ] + for path, receipt_contents in receipts.items(): + assert path.read_bytes() == receipt_contents + obligation = next(item for item in coverage["deferred"] if item["reason"] == deferred["reason"]) + assert obligation["provenance"] == {**descriptions, "workerId": worker_id, "attempt": 1} + assert sum(item["reason"] == deferred["reason"] for item in coverage["deferred"]) == 1 + for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"): + assert len({item["id"] for item in coverage[field]}) == len(coverage[field]) + for field in ("explicitExclusions", "openQuestions"): + assert len(coverage[field]) == 1 + item = coverage[field][0] + assert item["id"] == f"{worker_id}-attempt-1-{field}-1" + assert item["provenance"] == {**descriptions, "workerId": worker_id, "attempt": 1} + if explicit_ids: + assert obligation["surfaceIds"] == [item["id"] for item in actual] + assert {"workerId": worker_id, "attempt": 1, "completeness": "partial"} in coverage["reviews"] + report = (scan.scan_dir / "report.md").read_text() + for surface in surfaces: + for field in ("notes", "reason"): + if field in surface: + assert surface[field] in report + assert deferred["reason"] in report + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert tuple(workbench_db.execute(counter_query).fetchone()) == counters + attempt = workbench_db.execute( + "SELECT accepted_result_path, accepted_result_sha256 FROM deep_scan_attempts " + "WHERE worker_id = ?", + (worker_id,), + ).fetchone() + assert tuple(attempt) == (str(accepted), digest) + assert accepted.read_bytes() == contents + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + assert accepted.read_bytes() == contents diff --git a/plugins/codex-security/tests/test_budget_finding_identity.py b/plugins/codex-security/tests/test_budget_finding_identity.py new file mode 100644 index 000000000..44bcaec82 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_finding_identity.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes +from test_workbench_db import BUDGET_COST +from workbench_test_support import run_workbench + + +def prepare_budget_publication(connection, scan): + _, accepted, _ = accept_reducer(connection, scan) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with connection: + recipe = json.loads(connection.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + connection.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + connection.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL" + ) + return accepted, Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None + ) + + +@pytest.mark.parametrize("identity_kind", ["omitted", "candidate", "authored", "anchor-only"]) +def test_budget_publication_identifies_accepted_semantic_findings( + workbench_api, workbench_db, publication_scan, identity_kind +): + scan = publication_scan() + finding = scan.findings[0] + finding["title"] = "Archive extraction crosses output boundary" + finding.pop("identity", None) + finding.pop("extensions", None) + expected = {"anchor": "archive-extraction-crosses-output-boundary"} + if identity_kind == "candidate": + finding["extensions"] = {"candidateId": "archive-candidate"} + expected = {"anchor": "archive-candidate"} + elif identity_kind == "authored": + finding["identity"] = {"anchor": "authored-anchor", "instance": "first-route"} + expected = finding["identity"].copy() + elif identity_kind == "anchor-only": + finding["identity"] = expected = {"anchor": "authored-anchor"} + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(published) == 1 + assert published[0]["identity"] == expected + assert published[0]["codeEvidence"] == finding["codeEvidence"] + assert published[0]["remediation"] == finding["remediation"] + assert accepted.read_bytes() == original + selection = json.loads( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + ) + assert selection["resultSha256"] == hashlib.sha256(original).hexdigest() + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + + +def add_sibling_repairs(scan, count=2): + first = scan.findings[0] + first["title"] = "Shared repair" + first.pop("identity", None) + first.pop("extensions", None) + scan.findings.extend(copy.deepcopy(first) for _ in range(count - 1)) + for index, finding in enumerate(scan.findings, 1): + finding["locations"] = [{"path": "subdir/extract.py", "startLine": index}] + finding["provenance"]["sourceFindingIds"] = [f"review-{index}:candidate-{index}"] + finding["provenance"]["sourceFindings"][0]["id"] = f"review-{index}:candidate-{index}" + finding["codeEvidence"][0].update( + startLine=index, endLine=index, code=f"repair_point_{index}()" + ) + finding["remediation"] = f"Apply distinct repair {index} at this location." + finding["remediationTests"] = [f"Check distinct repair {index}."] + + +@pytest.mark.parametrize( + "case", + [ + "generated", + "candidate", + "authored-first", + "authored-last", + "reserved-instance", + "authored-siblings", + "different-rules", + "different-titles", + ], +) +def test_budget_publication_keeps_generated_sibling_repairs( + workbench_api, workbench_db, publication_scan, case +): + scan = publication_scan() + add_sibling_repairs(scan, 4 if case == "reserved-instance" else 2) + if case == "candidate": + for finding in scan.findings: + finding["extensions"] = {"candidateId": "shared-candidate"} + elif case in {"authored-first", "authored-last"}: + scan.findings[0 if case == "authored-first" else 1]["identity"] = { + "anchor": "shared-repair" + } + elif case == "reserved-instance": + scan.findings[2]["identity"] = {"anchor": "shared-repair"} + scan.findings[3]["identity"] = {"anchor": "shared-repair", "instance": "saved-2"} + elif case == "authored-siblings": + for index, finding in enumerate(scan.findings): + finding["identity"] = {"anchor": "shared-repair", "instance": f"route-{index}"} + elif case == "different-rules": + scan.findings[1]["ruleId"] = "distinct-rule" + elif case == "different-titles": + scan.findings[1]["title"] = "Distinct repair" + originals = copy.deepcopy(scan.findings) + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(published) == len(scan.findings) + assert len({finding["occurrenceId"] for finding in published}) == len(scan.findings) + for expected, actual in zip(scan.findings, published, strict=True): + for key in expected.keys() - {"findingId", "occurrenceId", "fingerprints"}: + assert actual[key] == expected[key] + if "identity" not in expected: + anchor = ( + "shared-candidate" + if case == "candidate" + else expected["title"].lower().replace(" ", "-") + ) + assert actual["identity"]["anchor"] == anchor + assert expected["remediation"] in (scan.scan_dir / "report.md").read_text() + if case in {"different-rules", "different-titles"}: + assert "instance" not in actual["identity"] + selection = workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + assert json.loads(selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection + ) + assert accepted.read_bytes() == original + assert scan.findings == originals + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + + +@pytest.mark.parametrize("identity", [{}, {"instance": "first-route"}, None]) +def test_budget_publication_does_not_repair_authored_invalid_identity( + workbench_api, workbench_db, publication_scan, identity +): + scan = publication_scan() + scan.findings[0]["identity"] = identity + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + with pytest.raises(SystemExit, match="identity"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert accepted.read_bytes() == original + assert scan.findings[0]["identity"] == identity + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] != "complete" + + +def test_budget_publication_rejects_duplicate_authored_identities( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + add_sibling_repairs(scan) + for finding in scan.findings: + finding["identity"] = {"anchor": "shared-repair", "instance": "authored-route"} + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + with pytest.raises(SystemExit, match="duplicate occurrence identity"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert accepted.read_bytes() == original + + +@pytest.mark.parametrize("boundary", ["before-selection-commit", "after-selection-commit"]) +def test_budget_sibling_selection_replays_in_a_fresh_process( + workbench_api, workbench_db, publication_scan, tmp_path, monkeypatch, boundary +): + scan = publication_scan() + add_sibling_repairs(scan) + accepted, args = prepare_budget_publication(workbench_db, scan) + original = accepted.read_bytes() + budget = workbench_api["complete_budget_exhausted_scan"] + drafts = [] + + def interrupt(*args, **kwargs): + drafts.append(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) + raise RuntimeError("Synthetic publication interruption.") + + state = tmp_path / "state" + state.mkdir(exist_ok=True) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + with monkeypatch.context() as patch: + if boundary == "before-selection-commit": + patch.setattr(workbench_api["deep_scan"], "cancel_active_workers", interrupt) + else: + patch.setitem(budget.__globals__, "complete_scan_locked", interrupt) + with pytest.raises(RuntimeError, match="Synthetic publication interruption"): + budget(connection, args) + selection = connection.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + if boundary == "before-selection-commit": + assert selection is None + assert not (scan.scan_dir / "findings.json").exists() + else: + assert json.loads(selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + run_workbench( + state, + "complete-budget-exhausted-scan", + "--scan-id", + scan.scan_id, + "--cost-json", + args.cost_json, + ) + published = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert [finding["identity"] for finding in published] == [ + finding["identity"] for finding in drafts[0] + ] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + replayed_selection = connection.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + if selection is not None: + assert replayed_selection == selection + assert ( + json.loads(replayed_selection)["resultSha256"] == hashlib.sha256(original).hexdigest() + ) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + before = published_bytes(scan) + run_workbench(state, "complete-scan", "--scan-id", scan.scan_id) + assert published_bytes(scan) == before + assert accepted.read_bytes() == original diff --git a/plugins/codex-security/tests/test_budget_report_evidence.py b/plugins/codex-security/tests/test_budget_report_evidence.py new file mode 100644 index 000000000..ca2803aad --- /dev/null +++ b/plugins/codex-security/tests/test_budget_report_evidence.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +from argparse import Namespace + +import pytest +from test_budget_selection_publication import accept_unmerged +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_db import BUDGET_COST + + +@pytest.mark.parametrize("accepted", [False, True], ids=["unfinished-audit", "accepted-unmerged"]) +def test_empty_budget_report_does_not_deny_saved_validation( + workbench_api, workbench_db, publication_scan, accepted +): + scan = publication_scan() + if accepted: + evidence = accept_unmerged(workbench_db, scan) + else: + result = add_worker(workbench_db, scan, status="running") + checkpoints = result.parent / "checkpoints" + checkpoints.mkdir() + evidence = checkpoints / ("a" * 64 + ".json") + evidence.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": False, + "findings": scan.findings, + "coverage": {**scan.coverage, "completeness": "partial"}, + } + ) + ) + (result.parent / "checkpoint-head.json").write_text( + json.dumps({"checkpoint": evidence.name}) + ) + original = evidence.read_bytes() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL" + ) + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + assert evidence.read_bytes() == original + assert json.loads(original)["findings"] + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert json.loads((scan.scan_dir / "coverage.json").read_text())["completeness"] == "partial" + report = (scan.scan_dir / "report.md").read_text() + assert "No findings were validated" not in report + assert "No findings are included in this partial report." in report + assert "Review the unresolved work in Open Questions And Follow Up." in report diff --git a/plugins/codex-security/tests/test_budget_selection_publication.py b/plugins/codex-security/tests/test_budget_selection_publication.py new file mode 100644 index 000000000..39da80b94 --- /dev/null +++ b/plugins/codex-security/tests/test_budget_selection_publication.py @@ -0,0 +1,802 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, saved_selection +from test_workbench_db import BUDGET_COST + + +def accept_unmerged(connection, scan): + result = add_worker(connection, scan) + worker_id = result.parent.name + result.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": { + **scan.coverage, + "completeness": "partial", + "deferred": [ + { + "id": "shared-gap", + "reason": "Independent unresolved review.", + "paths": ["subdir/extract.py"], + } + ], + }, + } + ) + ) + digest = hashlib.sha256(result.read_bytes()).hexdigest() + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(result.read_bytes()) + with connection: + connection.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", (worker_id,) + ) + connection.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at, " + "completed_at, accepted_result_path, accepted_result_sha256) VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan.scan_id, worker_id, scan.timestamp, scan.timestamp, str(accepted), digest), + ) + return accepted + + +@pytest.mark.parametrize("defect", ["changed-bytes", "incomplete-result"]) +def test_cost_before_selection_does_not_replace_invalid_accepted_evidence( + workbench_api, workbench_db, publication_scan, defect +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + document = json.loads(accepted.read_bytes()) + document["complete"] = False + accepted.write_text(json.dumps(document)) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', finalization_input_json = NULL" + ) + if defect == "incomplete-result": + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_sha256 = ?", + (hashlib.sha256(accepted.read_bytes()).hexdigest(),), + ) + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=None), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + assert ( + workbench_db.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + is None + ) + + +@pytest.mark.parametrize("accepted_kind", ["none", "unmerged", "reducer"]) +@pytest.mark.parametrize("cancel_first", [False, True]) +@pytest.mark.parametrize("lower_bound", [False, True]) +def test_cost_before_selection_retains_only_merged_findings( + workbench_api, workbench_db, publication_scan, accepted_kind, cancel_first, lower_bound +): + scan = publication_scan() + accepted = None + if accepted_kind == "reducer": + _, accepted, _ = accept_reducer(workbench_db, scan) + elif accepted_kind == "unmerged": + accepted = accept_unmerged(workbench_db, scan) + active = add_worker(workbench_db, scan, status="running") + active_id = active.parent.name + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET completed_at = NULL WHERE id = ?", (active_id,) + ) + workbench_db.execute( + "INSERT INTO deep_scan_attempts (scan_id, worker_id, attempt, status, started_at) " + "VALUES (?, ?, 1, 'running', ?)", + (scan.scan_id, active_id, scan.timestamp), + ) + counters = tuple( + workbench_db.execute( + "SELECT discovery_runs_dispatched, consecutive_no_new, completion_sequence FROM deep_scan_runs" + ).fetchone() + ) + original = accepted.read_bytes() if accepted is not None else None + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', " + "workflow_version = 'deep-security-scan/v2', manifest_path = NULL, " + "terminal_reason = NULL, completed_at = NULL, max_discovery_runs = 100" + ) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps({"lowerBound": BUDGET_COST} if lower_bound else BUDGET_COST), + message="Scan reached its original cost limit.", + ) + if cancel_first: + workbench_api["cancel_scan"](workbench_db, Namespace(scan_id=scan.scan_id, thread_id=None)) + before = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + assert published_bytes(scan) == before + else: + workbench_api["complete_budget_exhausted_scan"](workbench_db, args) + row = workbench_db.execute("SELECT * FROM scans").fetchone() + run = workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone() + assert row["status"] == "complete" + stored = json.loads(row["cost_json"]) + if lower_bound: + assert "cost" not in stored and "estimatedUsd" not in stored + assert stored["usage"]["coverage"] == "unavailable" + else: + assert stored == BUDGET_COST + assert run["terminal_reason"] == "capped" + assert run["error_message"] == args.message + assert args.message in json.loads(row["completion_warnings_json"]) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == (1 if accepted_kind == "reducer" else 0) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + if accepted_kind == "reducer": + selection = json.loads(run["finalization_input_json"]) + assert selection["resultSha256"] == hashlib.sha256(original).hexdigest() + assert any(item["id"] == "accepted-follow-up" for item in coverage["deferred"]) + assert ( + tuple( + workbench_db.execute( + "SELECT discovery_runs_dispatched, consecutive_no_new, completion_sequence FROM deep_scan_runs" + ).fetchone() + ) + == counters + ) + interrupted = workbench_db.execute( + "SELECT * FROM deep_scan_attempts WHERE worker_id = ?", (active_id,) + ).fetchone() + assert interrupted["status"] == "canceled" + assert interrupted["end_reason"] == "scan_stopped" + if accepted_kind == "unmerged": + assert any( + item.get("provenance", {}).get("sourceId") == "shared-gap" + for item in coverage["deferred"] + ) + before = published_bytes(scan) + workbench_api["complete_scan"]( + workbench_db, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert published_bytes(scan) == before + if accepted is not None: + assert accepted.read_bytes() == original + + +@pytest.mark.parametrize( + "bound", [None, {**BUDGET_COST, "estimatedUsd": -1}, {**BUDGET_COST, "estimatedUsd": 0.005}] +) +def test_lower_bound_budget_rejects_invalid_or_unexceeded_cost( + workbench_api, workbench_db, publication_scan, bound +): + scan = publication_scan() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps({"lowerBound": bound}), message=None + ), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +def test_lower_bound_is_not_an_ordinary_completion_cost( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + before = published_bytes(scan) + with pytest.raises(SystemExit): + workbench_api["complete_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=json.dumps({"lowerBound": BUDGET_COST}), + ), + ) + assert published_bytes(scan) == before + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +def test_budget_keeps_selection_committed_before_transaction_and_unmerged_obligations( + workbench_api, workbench_db, publication_scan, reason +): + scan = publication_scan() + _, accepted, original_coverage = accept_reducer(workbench_db, scan) + unmerged = [accept_unmerged(workbench_db, scan) for _ in range(2)] + originals = {path: path.read_bytes() for path in [accepted, *unmerged]} + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + # The selected publication may commit before cost recovery reaches its + # transaction. Budget completion must use that exact input and cause. + committed = saved_selection(workbench_db, scan, accepted, reason=reason) + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ), + ) + run = workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone() + assert json.loads(run["finalization_input_json"]) == committed + assert run["terminal_reason"] == reason + assert all(path.read_bytes() == contents for path, contents in originals.items()) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert original_coverage["deferred"][0] in coverage["deferred"] + gaps = [ + item + for item in coverage["deferred"] + if item.get("provenance", {}).get("sourceId") == "shared-gap" + ] + assert len(gaps) == 2 + assert len({item["id"] for item in gaps}) == 2 + assert {item["provenance"]["workerId"] for item in gaps} == { + path.parent.parent.name for path in unmerged + } + assert all(item["provenance"]["attempt"] == 1 for item in gaps) + + +def test_budget_accepts_complete_reducer_without_optional_complete_flag( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + document = json.loads(accepted.read_bytes()) + del document["complete"] + accepted.write_text(json.dumps(document)) + digest = hashlib.sha256(accepted.read_bytes()).hexdigest() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute("UPDATE deep_scan_attempts SET accepted_result_sha256 = ?", (digest,)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ), + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert hashlib.sha256(accepted.read_bytes()).hexdigest() == digest + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + + +def test_budget_preserves_accepted_scope_and_unmerged_scope_limits( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + scope = { + "summary": "Synthetic upload boundaries.", + "artifactsReviewed": ["subdir/extract.py"], + "runtimeStatus": "Synthetic runtime unavailable.", + "validationMode": "Static validation of the accepted findings.", + "context": "External deployment behavior remains unresolved.", + "limitations": ["The accepted review did not execute the deployment integration."], + } + unmerged_limit = "A separate unmerged review could not inspect the deployment credentials." + for path, value in [(accepted, scope), (unmerged, {"limitations": [unmerged_limit]})]: + document = json.loads(path.read_bytes()) + document["scope"] = value + path.write_text(json.dumps(document)) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_sha256 = ? WHERE accepted_result_path = ?", + (hashlib.sha256(path.read_bytes()).hexdigest(), str(path)), + ) + originals = {p: p.read_bytes() for p in [accepted, unmerged]} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + warning = "Scan stopped at its original cost limit before further validation." + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=warning), + ) + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + saved_scope = manifest["scan"]["scope"] + for key, value in scope.items(): + if key == "limitations": + assert all(item in saved_scope[key] for item in value) + else: + assert saved_scope[key] == value + assert warning in saved_scope["limitations"] + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any( + item["reason"] == unmerged_limit + and item["provenance"]["workerId"] == unmerged.parent.parent.name + for item in coverage["deferred"] + ) + assert all(p.read_bytes() == contents for p, contents in originals.items()) + assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 1 + + +@pytest.mark.parametrize("has_reducer", [False, True]) +def test_budget_replay_after_draft_commit_keeps_unmerged_coverage_once( + workbench_api, workbench_db, publication_scan, tmp_path, monkeypatch, has_reducer +): + scan = publication_scan() + if has_reducer: + accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + original = unmerged.read_bytes() + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + budget = workbench_api["complete_budget_exhausted_scan"] + original_complete = budget.__globals__["complete_scan_locked"] + + def fail_before_seal(*args, **kwargs): + raise RuntimeError("Synthetic process loss after budget draft commit.") + + monkeypatch.setitem(budget.__globals__, "complete_scan_locked", fail_before_seal) + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original cost interruption.", + ) + path = tmp_path / "budget-gap.sqlite3" + with sqlite3.connect(path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + with pytest.raises(RuntimeError, match="after budget draft commit"): + budget(connection, args) + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == "succeeded" + selection = run["finalization_input_json"] + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "running" + before = json.loads((scan.scan_dir / "coverage.json").read_text()) + monkeypatch.setitem(budget.__globals__, "complete_scan_locked", original_complete) + with sqlite3.connect(path) as connection: + connection.row_factory = sqlite3.Row + budget(connection, args) + assert connection.execute("SELECT status FROM scans").fetchone()[0] == "complete" + assert ( + connection.execute("SELECT finalization_input_json FROM deep_scan_runs").fetchone()[0] + == selection + ) + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + for key in ("surfaces", "explicitExclusions", "deferred", "reviews"): + assert coverage.get(key, []) == before.get(key, []) + assert len({row["id"] for row in coverage["deferred"]}) == len(coverage["deferred"]) + assert unmerged.read_bytes() == original + + +@pytest.mark.parametrize("selected", [False, True], ids=["legacy-v1", "selected-v2"]) +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +@pytest.mark.parametrize("cancel_first", [False, True], ids=["budget-first", "cancel-first"]) +def test_budget_completion_and_cancel_keep_the_committed_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, selected, reason, cancel_first +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, reason=reason) if selected else None + scan.coverage = coverage + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute("UPDATE deep_scan_runs SET terminal_reason = ?", (reason,)) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + database_path = tmp_path / "budget-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + workbench_api["write_scan_draft"](connection, staged) + accepted_bytes = accepted.read_bytes() + warning = "Scan stopped after reaching its configured cost limit." + budget_args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message=warning + ) + cancel_args = Namespace(scan_id=scan.scan_id, thread_id=None) + if cancel_first: + workbench_api["cancel_scan"](connection, cancel_args) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + else: + workbench_api["complete_budget_exhausted_scan"](connection, budget_args) + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="running"): + workbench_api["cancel_scan"](connection, cancel_args) + assert published_bytes(scan) == frozen + + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + row = connection.execute("SELECT * FROM scans").fetchone() + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert row["status"] == ("failed" if cancel_first else "complete") + assert bool(row["canceled_at"]) == cancel_first + assert run["terminal_reason"] == reason + assert json.loads(run["finalization_input_json"] or "null") == selection + assert accepted.read_bytes() == accepted_bytes + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert any(item["id"] == "accepted-follow-up" for item in coverage["deferred"]) + if not cancel_first: + assert any(item["id"] == "scan-cost-limit" for item in coverage["deferred"]) + assert warning in json.loads(row["completion_warnings_json"]) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + with pytest.raises(SystemExit, match="stopped"): + workbench_api["write_scan_draft"](connection, staged) + assert published_bytes(scan) == frozen + + +@pytest.mark.parametrize( + "cut", ["after-findings", "after-coverage", "after-draft-commit", "after-seal"] +) +@pytest.mark.parametrize("has_reducer", [False, True]) +def test_budget_process_death_replays_exact_evidence( + workbench_api, workbench_db, publication_scan, tmp_path, cut, has_reducer +): + scan = publication_scan() + thread_id = "a23e657b-c14c-4da7-bd20-baa9e7579390" + workbench_api["set_scan_thread"]( + workbench_db, Namespace(scan_id=scan.scan_id, thread_id=thread_id) + ) + accepted = None + if has_reducer: + _, accepted, _ = accept_reducer(workbench_db, scan) + unmerged = accept_unmerged(workbench_db, scan) + originals = {str(p): p.read_bytes() for p in [unmerged, *([accepted] if accepted else [])]} + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / name).unlink() + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status='running', workflow_version='deep-security-scan/v2'" + ) + database = tmp_path / "budget-process-cut.sqlite3" + with sqlite3.connect(database) as db: + workbench_db.backup(db) + budget = workbench_api["complete_budget_exhausted_scan"] + args = Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original synthetic cost stop.", + ) + marker = tmp_path / "crash-marker.json" + child_source = """ +from argparse import Namespace +from pathlib import Path +import json +import os +import runpy +import sqlite3 +import sys + +script, database, scan_id, cost_json, warning, cut, marker = sys.argv[1:] +api = runpy.run_path(str(script), run_name='fault_review_budget_child') +budget = api['complete_budget_exhausted_scan'] +deep = budget.__globals__['deep_scan'] +deep.configure(deep.DeepScanDependencies(**{ + name: api['preserve_stopped_results_after_transition' if name == 'preserve_stopped_results' else name] + for name in deep.DeepScanDependencies.__dataclass_fields__ +})) + + +def terminate(stage): + Path(marker).write_text(json.dumps({'stage': stage, 'pid': os.getpid()})) + os._exit(86) + + +if cut == 'after-draft-commit': + budget.__globals__['complete_scan_locked'] = lambda *a, **kw: terminate(cut) +elif cut == 'after-seal': + original = budget.__globals__['_write_prepared_scan_finalization'] + + def seal_then_die(prepared): + original(prepared) + terminate(cut) + + budget.__globals__['_write_prepared_scan_finalization'] = seal_then_die +else: + saved = budget.__globals__['saved_results'] + original = saved.write_scan_local_bytes + + def write_then_die(scan_dir, name, contents): + result = original(scan_dir, name, contents) + if name == ('findings.json' if cut == 'after-findings' else 'coverage.json'): + terminate(cut) + return result + + saved.write_scan_local_bytes = write_then_die + +with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute('PRAGMA foreign_keys=ON') + budget(connection, Namespace(scan_id=scan_id, cost_json=cost_json, message=warning)) +os._exit(87) +""" + child = subprocess.run( + [ + sys.executable, + "-c", + child_source, + workbench_api["__file__"], + str(database), + scan.scan_id, + args.cost_json, + args.message, + cut, + str(marker), + ], + capture_output=True, + text=True, + ) + assert child.returncode == 86, (child.returncode, child.stdout, child.stderr) + assert json.loads(marker.read_text())["stage"] == cut + before = { + name: (scan.scan_dir / name).exists() + for name in ("findings.json", "coverage.json", "scan-manifest.json") + } + sealed_bytes = published_bytes(scan) if cut == "after-seal" else None + if sealed_bytes is not None: + assert json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"]["sealedAt"] + with sqlite3.connect(database) as db: + db.row_factory = sqlite3.Row + db.execute("PRAGMA foreign_keys=ON") + run = dict(db.execute("SELECT * FROM deep_scan_runs").fetchone()) + snapshot = { + "cut": cut, + "hasReducer": has_reducer, + "database": str(database), + "scanDir": str(scan.scan_dir), + "beforeFiles": before, + "beforeStatus": run["status"], + "beforeSelection": run["finalization_input_json"], + } + saved_scan = workbench_api["require_scan"](db, scan.scan_id) + resumed = workbench_api["scan_history"].cli_scan_resume( + db, + saved_scan, + workbench_api["require_workspace"](db, saved_scan["workspace_id"]), + **{ + name: workbench_api[source] + for name, source in { + "parse_scan_recipe": "parse_scan_recipe", + "scan_contract": "scan_contract", + "require_scan_directory": "require_canonical_scan_directory", + "artifact_path": "artifact_path", + "read_json_object": "read_json_object", + "workbench_completion_binding": "workbench_completion_binding", + }.items() + }, + ) + assert resumed["scanId"] == scan.scan_id + assert resumed["threadId"] == thread_id + try: + budget(db, args) + except BaseException as error: + snapshot["replayError"] = str(error) + (tmp_path / "crash-result.json").write_text(json.dumps(snapshot, indent=2)) + raise + snapshot["afterStatus"] = db.execute("SELECT status FROM scans").fetchone()[0] + snapshot["afterSelection"] = db.execute( + "SELECT finalization_input_json FROM deep_scan_runs" + ).fetchone()[0] + (tmp_path / "crash-result.json").write_text(json.dumps(snapshot, indent=2)) + assert snapshot["afterStatus"] == "complete" + if sealed_bytes is not None: + assert published_bytes(scan) == sealed_bytes + if run["finalization_input_json"] is not None: + assert snapshot["afterSelection"] == run["finalization_input_json"] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert len(findings) == int(has_reducer) + assert coverage["completeness"] == "partial" + assert len({x["id"] for x in coverage["deferred"]}) == len(coverage["deferred"]) + assert all(Path(path).read_bytes() == data for path, data in originals.items()) + + +@pytest.mark.parametrize("files", ["findings", "coverage", "manifest", "findings-and-coverage"]) +def test_budget_rejects_unrelated_incomplete_drafts( + workbench_api, workbench_db, publication_scan, files +): + scan = publication_scan() + accept_unmerged(workbench_db, scan) + keep = { + "findings": {"findings.json"}, + "coverage": {"coverage.json"}, + "manifest": {"scan-manifest.json"}, + "findings-and-coverage": {"findings.json", "coverage.json"}, + }[files] + for name in ("scan-manifest.json", "findings.json", "coverage.json"): + if name not in keep: + (scan.scan_dir / name).unlink() + before = {name: (scan.scan_dir / name).read_bytes() for name in keep} + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + with pytest.raises(SystemExit, match="incomplete canonical scan draft"): + workbench_api["complete_budget_exhausted_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + cost_json=json.dumps(BUDGET_COST), + message="Original cost stop.", + ), + ) + assert {name: (scan.scan_dir / name).read_bytes() for name in keep} == before + assert all( + not (scan.scan_dir / name).exists() + for name in ("scan-manifest.json", "findings.json", "coverage.json") + if name not in keep + ) + assert workbench_db.execute("SELECT status FROM scans").fetchone()[0] == "running" + + +@pytest.mark.parametrize( + "state", + ["changed-findings", "wrong-scan", "running-discovery", "canceled", "other-owner", "complete"], +) +def test_budget_sealed_replay_keeps_integrity_and_ownership_guards( + workbench_api, workbench_db, publication_scan, monkeypatch, state +): + scan = publication_scan() + accept_reducer(workbench_db, scan) + with workbench_db: + recipe = json.loads(workbench_db.execute("SELECT recipe_json FROM scans").fetchone()[0]) + recipe["maxCostUsd"] = 0.005 + workbench_db.execute("UPDATE scans SET recipe_json = ?", (json.dumps(recipe),)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', workflow_version = 'deep-security-scan/v2'" + ) + budget = workbench_api["complete_budget_exhausted_scan"] + complete = budget.__globals__["complete_scan_locked"] + args = Namespace( + scan_id=scan.scan_id, cost_json=json.dumps(BUDGET_COST), message="Original cost stop." + ) + with monkeypatch.context() as patch: + patch.setitem( + budget.__globals__, + "complete_scan_locked", + lambda *a: complete(*a, prepare_only=True), + ) + budget(workbench_db, args) + manifest = scan.scan_dir / "scan-manifest.json" + assert json.loads(manifest.read_text())["scan"]["sealedAt"] + if state in {"changed-findings", "wrong-scan"}: + path = scan.scan_dir / ( + "findings.json" if state == "changed-findings" else "scan-manifest.json" + ) + document = json.loads(path.read_text()) + if state == "changed-findings": + document["findings"] = [] + else: + document["scan"]["id"] = "69078890-d24c-4416-a6fa-c286825bef88" + path.write_text(json.dumps(document)) + elif state == "running-discovery": + with workbench_db: + workbench_db.execute("UPDATE deep_scan_runs SET status = 'running'") + elif state == "canceled": + workbench_api["cancel_scan"](workbench_db, Namespace(scan_id=scan.scan_id, thread_id=None)) + elif state == "other-owner": + with workbench_db: + workbench_db.execute( + "UPDATE scans SET handoff_claim_token = 'a3292ae4-9b47-430f-8ed6-73ff73db575c'" + ) + else: + complete(workbench_db, scan.scan_id, None, args.cost_json) + before_files = published_bytes(scan) + before_scan = dict(workbench_db.execute("SELECT * FROM scans").fetchone()) + before_run = dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) + with pytest.raises(SystemExit): + budget(workbench_db, args) + assert published_bytes(scan) == before_files + assert dict(workbench_db.execute("SELECT * FROM scans").fetchone()) == before_scan + assert dict(workbench_db.execute("SELECT * FROM deep_scan_runs").fetchone()) == before_run + + +@pytest.mark.parametrize( + "state", ["parent-canceled", "stopping", "failed", "canceled", "unselected"] +) +def test_budget_resume_keeps_explicit_stop_and_unselected_guards( + workbench_api, workbench_db, publication_scan, state +): + scan = publication_scan() + workbench_api["set_scan_thread"]( + workbench_db, + Namespace(scan_id=scan.scan_id, thread_id="a23e657b-c14c-4da7-bd20-baa9e7579390"), + ) + with workbench_db: + if state == "parent-canceled": + workbench_db.execute("UPDATE scans SET canceled_at = ?", (scan.timestamp,)) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = ?, cancel_requested = 1, finalization_input_json = NULL", + ( + "running" + if state == "stopping" + else state + if state in {"failed", "canceled"} + else "succeeded", + ), + ) + saved_scan = workbench_api["require_scan"](workbench_db, scan.scan_id) + with pytest.raises(SystemExit, match="cannot resume"): + workbench_api["scan_history"].cli_scan_resume( + workbench_db, + saved_scan, + workbench_api["require_workspace"](workbench_db, saved_scan["workspace_id"]), + **{ + name: workbench_api[source] + for name, source in { + "parse_scan_recipe": "parse_scan_recipe", + "scan_contract": "scan_contract", + "require_scan_directory": "require_canonical_scan_directory", + "artifact_path": "artifact_path", + "read_json_object": "read_json_object", + "workbench_completion_binding": "workbench_completion_binding", + }.items() + }, + ) diff --git a/plugins/codex-security/tests/test_checkpoint_publication_authority.py b/plugins/codex-security/tests/test_checkpoint_publication_authority.py new file mode 100644 index 000000000..1204e3cce --- /dev/null +++ b/plugins/codex-security/tests/test_checkpoint_publication_authority.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +import copy +import json +import uuid +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_workbench_standard_deep_results import deep_scan_fixture, worker_paths +from workbench_test_support import run_workbench, write_checkpoint, write_completed_contract + + +def test_public_stop_retains_accepted_partial_evidence_and_newer_rejection(tmp_path): + state, home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path, workers=2) + environment = {"CODEX_HOME": str(home)} + contract = tmp_path / "contract" + contract.mkdir() + write_completed_contract(contract, scan_id, target, relative_path="app.py") + finding = json.loads((contract / "findings.json").read_text())["findings"][0] + deferred = { + "candidateId": "pending-query", + "reason": "Validation is pending.", + "paths": ["app.py"], + } + coverage = { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [deferred], + } + workers = [] + for accepted in (True, False): + name = "accepted" if accepted else "interrupted" + prompt, output, result = worker_paths(scan_dir, name) + worker_id = str(uuid.uuid4()) + worker_args = ( + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + worker_id, + "--kind", + "discovery", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(output), + "--attempt", + "1", + ) + run_workbench(state, *worker_args, "--status", "running", environment=environment) + current = copy.deepcopy(finding) + current["identity"]["anchor"] = name + current["extensions"] = {"candidateId": name} + draft = {"scanId": scan_id, "complete": True, "findings": [current], "coverage": coverage} + result.write_text(json.dumps(draft)) + write_checkpoint(output / "checkpoints", draft) + if accepted: + run_workbench( + state, + *worker_args, + "--status", + "succeeded", + "--result-manifest-path", + str(result), + environment=environment, + ) + else: + rejected = { + **draft, + "complete": False, + "findings": [], + "coverage": { + **coverage, + "surfaces": [ + { + "candidateId": name, + "label": "Reviewed candidate", + "disposition": "rejected", + "receiptRefs": [], + } + ], + }, + } + head = write_checkpoint(output / "checkpoints", rejected) + (output / "checkpoint-head.json").write_text(json.dumps({"checkpoint": head.name})) + workers.append(worker_id) + + stopped = run_workbench( + state, + "fail-deep-scan", + "--scan-id", + scan_id, + "--message", + "Original worker failure.", + "--deep-status", + "interrupted", + environment=environment, + )["deepScan"] + assert stopped["status"] == "interrupted" + scan = run_workbench(state, "get-scan", "--scan-id", scan_id)["scan"] + assert scan["findingCount"] == 1 + findings = json.loads((scan_dir / "findings.json").read_text())["findings"] + assert [item["identity"]["anchor"] for item in findings] == ["accepted"] + retained_coverage = json.loads((scan_dir / "coverage.json").read_text()) + assert retained_coverage["completeness"] == "partial" + assert any(item.get("candidateId") == "pending-query" for item in retained_coverage["deferred"]) + assert any( + item.get("candidateId") == "interrupted" and item["disposition"] == "rejected" + for item in retained_coverage["surfaces"] + ) + assert scan["failureMessage"] == "Original worker failure." + assert {worker["id"]: worker["status"] for worker in stopped["workers"]} == { + workers[0]: "succeeded", + workers[1]: "canceled", + } + + +@pytest.mark.parametrize("archived", [False, True], ids=["current", "archived"]) +@pytest.mark.parametrize("has_head", [True, False], ids=["committed-head", "legacy"]) +@pytest.mark.parametrize("complete", [False, True], ids=["checkpoint", "complete"]) +def test_recovery_honors_rejection_committed_before_result_replacement( + workbench_api, workbench_db, publication_scan, archived, has_head, complete +): + scan = publication_scan() + provisional = copy.deepcopy(scan.findings[0]) + provisional["extensions"] = {"candidateId": "candidate-rejected"} + retained = copy.deepcopy(scan.findings[0]) + retained["identity"]["anchor"] = "independent-finding" + retained["extensions"] = {"candidateId": "candidate-retained"} + retained["locations"][0]["startLine"] = 20 + retained["locations"][0]["endLine"] = 21 + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result_path = add_worker(workbench_db, scan, status="canceled") + if archived: + result_path = result_path.parent / "attempts" / "attempt-1" / "result.json" + result_path.parent.mkdir(parents=True) + previous = { + "scanId": scan.scan_id, + "complete": True, + "findings": [provisional, retained], + "coverage": scan.coverage, + } + result_path.write_text(json.dumps(previous)) + old_checkpoint = write_checkpoint(result_path.parent / "checkpoints", previous) + rejected = { + **previous, + "complete": complete, + "findings": [retained], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-rejected", + "label": "Validated candidate disposition", + "disposition": "rejected", + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(result_path.parent / "checkpoints", rejected) + if has_head: + (result_path.parent / "checkpoint-head.json").write_text( + json.dumps({"checkpoint": checkpoint.name}) + ) + saved_bytes = {path: path.read_bytes() for path in (result_path, old_checkpoint, checkpoint)} + + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert stopped["findingCount"] == len(findings) == (1 if has_head else 2) + assert any(finding["identity"]["anchor"] == "independent-finding" for finding in findings) + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + if has_head: + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert any( + surface.get("candidateId") == "candidate-rejected" + and surface.get("disposition") == "rejected" + for surface in coverage["surfaces"] + ) + + +def save_disposition(scan, directory, disposition): + directory.mkdir(parents=True, exist_ok=True) + finding = copy.deepcopy(scan.findings[0]) + finding["extensions"] = {"candidateId": "candidate-disposition"} + draft = { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding] if disposition == "reported" else [], + "coverage": { + **scan.coverage, + "surfaces": [ + { + "candidateId": "candidate-disposition", + "label": "Validated candidate disposition", + "disposition": disposition, + "receiptRefs": [], + } + ], + }, + } + checkpoint = write_checkpoint(directory / "checkpoints", draft) + (directory / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + return draft + + +@pytest.mark.parametrize("archived", [False, True], ids=["current-head", "newer-archive"]) +@pytest.mark.parametrize("disposition", ["reported", "rejected"]) +def test_newer_checkpoint_disposition_precedes_older_archived_head( + workbench_api, workbench_db, publication_scan, archived, disposition +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + old = result.parent / "attempts" / "attempt-2" + save_disposition(scan, old, "rejected" if disposition == "reported" else "reported") + current = result.parent / "attempts" / "attempt-10" if archived else result.parent + draft = save_disposition(scan, current, disposition) + (current / "result.json").write_text(json.dumps(draft)) + + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["findingCount"] == (1 if disposition == "reported" else 0) + + +@pytest.mark.parametrize("head_change", ["replaced", "removed", "missing-checkpoint"]) +def test_frozen_stopped_replay_ignores_later_worker_head_changes( + workbench_api, workbench_db, publication_scan, monkeypatch, head_change +): + import finalize_scan_contract + + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + checkpoint_name = json.loads((result.parent / "checkpoint-head.json").read_text())["checkpoint"] + directory = result.parent.relative_to(scan.scan_dir).as_posix() + expected_heads = {directory: f"{directory}/checkpoints/{checkpoint_name}"} + original_outputs = { + name: (scan.scan_dir / name).read_bytes() + for name in ("findings.json", "coverage.json", "scan-manifest.json") + } + write_bytes = finalize_scan_contract.write_scan_local_bytes + failed_writes = [] + + def fail_coverage_write(directory, relative, payload, **kwargs): + if relative != "coverage.json" or failed_writes: + return write_bytes(directory, relative, payload, **kwargs) + # Exercise the real writer after findings have reached disk. Remove the + # temporary obstruction before the publisher restores its old outputs. + failed_writes.append(json.loads((directory / "findings.json").read_text())) + path = directory / relative + previous_bytes = path.read_bytes() + path.unlink() + path.mkdir() + try: + return write_bytes(directory, relative, payload, **kwargs) + finally: + path.rmdir() + path.write_bytes(previous_bytes) + + with monkeypatch.context() as patch: + patch.setattr(finalize_scan_contract, "write_scan_local_bytes", fail_coverage_write) + workbench_api["fail_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert len(failed_writes) == 1 + assert "scanId" in failed_writes[0] + assert row["status"] == "failed" + assert row["failure_message"] == "Audit stopped." + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + assert all( + (scan.scan_dir / name).read_bytes() == contents + for name, contents in original_outputs.items() + ) + original_sources = row["retained_source_digests_json"] + original_run = dict( + workbench_db.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + ) + head = result.parent / "checkpoint-head.json" + if head_change == "replaced": + save_disposition(scan, result.parent, "reported") + elif head_change == "removed": + head.unlink() + else: + head.write_text(json.dumps({"checkpoint": "a" * 64 + ".json"})) + + replayed = workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + assert replayed["findingCount"] == 0 + assert json.loads((scan.scan_dir / "findings.json").read_text())["findings"] == [] + assert json.loads(result.read_text()) == previous + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["failure_message"] == "Audit stopped." + assert row["retained_source_digests_json"] == original_sources + assert json.loads(row["retained_checkpoint_heads_json"]) == expected_heads + assert row["seal_manifest_digest"] + manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text()) + assert manifest["scan"]["preservedCheckpointHeads"] == expected_heads + assert ( + dict( + workbench_db.execute( + "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + ) + == original_run + ) + + +def test_explicit_recovery_observes_head_change_between_existing_checkpoints( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + assert stopped["findingCount"] == 0 + + save_disposition(scan, result.parent, "reported") + + context = workbench_api["scan_context"](workbench_db, scan.scan_id)["scan"] + assert context["resultsRecoveryNeeded"] is True + recovered = workbench_api["recover_scan_results"]( + workbench_db, Namespace(scan_id=scan.scan_id) + )["scan"] + assert recovered["findingCount"] == 1 + assert recovered["resultsRecoveryNeeded"] is False + + +def test_legacy_frozen_publication_keeps_result_fallback_without_saved_heads( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + previous = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(previous)) + save_disposition(scan, result.parent, "rejected") + (result.parent / "checkpoint-head.json").unlink() + + def fail_before_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + fail_before_publication, + ) + workbench_api["fail_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + ) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET retained_checkpoint_heads_json = NULL WHERE id = ?", (scan.scan_id,) + ) + save_disposition(scan, result.parent, "rejected") + + replayed = workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + )["scan"] + + assert replayed["findingCount"] == 1 diff --git a/plugins/codex-security/tests/test_checkpoint_schema.py b/plugins/codex-security/tests/test_checkpoint_schema.py new file mode 100644 index 000000000..bb471267c --- /dev/null +++ b/plugins/codex-security/tests/test_checkpoint_schema.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +import sqlite3 +from contextlib import closing + +import pytest + + +@pytest.mark.parametrize("upgrade", [False, True], ids=["fresh", "upgrade"]) +def test_frozen_checkpoint_head_migration_preserves_scan_state(workbench_api, upgrade): + migrations = workbench_api["MIGRATIONS"] + timestamp = "2026-07-01T00:00:00Z" + + def migrate(connection, selected): + workbench_api["apply_schema_migrations"]( + connection, selected, lambda: timestamp, workbench_api["backfill_security_targets"] + ) + + with closing(sqlite3.connect(":memory:")) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + migrate( + connection, + tuple(item for item in migrations if item[0] != 47) if upgrade else migrations, + ) + connection.execute( + "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)", + ("workspace", timestamp, timestamp), + ) + for status in ("running", "failed"): + connection.execute( + "INSERT INTO scans (id, workspace_id, target_path, target_revision, scope, " + "mode, scan_dir, status, phase, started_at, created_at, updated_at, " + "failure_message, retained_source_digests_json) " + "VALUES (?, 'workspace', 'target', 'revision', '.', 'deep', ?, ?, " + "'discovery', ?, ?, ?, ?, ?)", + ( + status, + f"scans/{status}", + status, + timestamp, + timestamp, + timestamp, + "Original stop reason." if status == "failed" else None, + json.dumps({"workers/review/result.json": "b" * 64}) + if status == "failed" + else None, + ), + ) + before = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")] + migrate(connection, migrations) + after = [dict(row) for row in connection.execute("SELECT * FROM scans ORDER BY id")] + for original, updated in zip(before, after, strict=True): + assert updated.pop("retained_checkpoint_heads_json") is None + original.pop("retained_checkpoint_heads_json", None) + assert updated == original + + heads = json.dumps({"workers/review": "workers/review/checkpoints/" + "a" * 64 + ".json"}) + connection.execute( + "UPDATE scans SET retained_checkpoint_heads_json = ? WHERE id = 'failed'", (heads,) + ) + migrate(connection, migrations) + assert ( + connection.execute( + "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'failed'" + ).fetchone()[0] + == heads + ) + assert ( + connection.execute( + "SELECT retained_checkpoint_heads_json FROM scans WHERE id = 'running'" + ).fetchone()[0] + is None + ) + assert [ + tuple(row) + for row in connection.execute( + "SELECT version, name FROM schema_migrations WHERE version = 47" + ) + ] == [(47, "freeze stopped scan checkpoint selections")] + assert connection.execute("PRAGMA foreign_key_check").fetchall() == [] diff --git a/plugins/codex-security/tests/test_deep_scan_compatibility.py b/plugins/codex-security/tests/test_deep_scan_compatibility.py new file mode 100644 index 000000000..96b01a874 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_compatibility.py @@ -0,0 +1,380 @@ +"""Compatibility checks use real persisted scans and preserve unsupported state.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path + +import pytest +from workbench_test_support import SCRIPT, run_workbench + + +def snapshot(state_dir: Path) -> str: + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + return "\n".join(connection.iterdump()) + + +def claim_requiring_original_settings( + state: Path, scan_id: str +) -> subprocess.CompletedProcess[str]: + # Exercise the private MCP requirement against both the parent and fixed + # workbench, without adding a public CLI argument. + return subprocess.run( + [ + sys.executable, + "-c", + "\n".join( + [ + "import runpy, sys", + "script = sys.argv.pop(1)", + "main = runpy.run_path(script)['main']", + "namespace = main.__globals__", + "parse = namespace['parse_args']", + "def parse_with_requirement(*args, **kwargs):", + " result = parse(*args, **kwargs)", + " result.require_execution_settings = True", + " return result", + "namespace['parse_args'] = parse_with_requirement", + "main()", + ] + ), + str(SCRIPT), + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(state)}, + capture_output=True, + text=True, + timeout=30, + ) + + +@pytest.mark.parametrize( + "workflow,settings_version", + [ + ("deep-security-scan/v1", 99), + ("deep-scan-mcp/v1", 99), + ("deep-security-scan/v2", 99), + ("deep-security-scan/v2", None), + ], +) +def test_missing_or_unsupported_settings_reject_before_takeover( + tmp_path: Path, workflow: str, settings_version: int | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + workflow, + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 2, " + "phase = 'discovery', updated_at = '2000-01-01T00:00:00Z'" + ) + settings_path = Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json" + saved = None + if settings_version is not None: + settings_path.parent.mkdir(parents=True, exist_ok=True) + saved = json.dumps( + { + "version": settings_version, + "settings": {"codexPath": "/fixture/codex", "codexHome": "/fixture/home"}, + } + ).encode() + settings_path.write_bytes(saved) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert snapshot(state) == before, ( + "settings rejection must precede ownership and worker recovery" + ) + assert result.returncode != 0 + assert "execution settings" in result.stderr + assert (settings_path.read_bytes() if settings_path.exists() else None) == saved + + +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_legacy_takeover_does_not_require_or_create_a_new_snapshot( + tmp_path: Path, workflow: str +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + workflow, + )["deepScan"] + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout)["deepScan"] + assert observed["workflowVersion"] == workflow + assert observed["config"] == run["config"] + assert not (Path(run["scanDir"]) / "artifacts/deep_discovery/execution-settings.json").exists() + + +@pytest.mark.parametrize("completion_only", [False, True]) +def test_observation_and_selected_completion_do_not_require_worker_settings( + tmp_path: Path, completion_only: bool +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + claim = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + run["scanId"], + "--thread-id", + "fixture-thread", + ) + if completion_only: + selection = { + "version": 1, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": run["createdAt"], + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET finalization_input_json = ?, " + "updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selection),), + ) + before = snapshot(state) + result = claim_requiring_original_settings(state, run["scanId"]) + assert result.returncode == 0, result.stderr + observed = json.loads(result.stdout) + if completion_only: + assert observed["coordinatorDisposition"] == "adopted" + assert observed["deepScan"]["finalizationInput"] == selection + else: + assert observed["coordinatorDisposition"] == "observing" + assert ( + observed["deepScan"]["coordinatorGeneration"] + == claim["deepScan"]["coordinatorGeneration"] + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize( + "version", ["deep-security-scan/v1", "deep-scan-mcp/v1", "deep-security-scan/v2"] +) +def test_supported_workflows_keep_their_identity(tmp_path: Path, version: str) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--workflow-version", + version, + )["deepScan"] + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["workflowVersion"] == version + assert claimed["schemaVersion"] == 1 + assert claimed["noNewStreak"] == begun["noNewStreak"] + assert claimed["config"] == begun["config"] + + +@pytest.mark.parametrize("field,value", [("workflow_version", "future/v99")]) +@pytest.mark.parametrize("operation", ["begin", "claim", "handoff"]) +def test_unsupported_execution_does_not_mutate( + tmp_path: Path, + field: str, + value: str | int, + operation: str, +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + scan_id = str(begun["scanId"]) + if operation == "claim": + artifact_dir = Path(str(begun["scanDir"])) / "artifacts" / "deep_discovery" / "worker" + artifact_dir.mkdir(parents=True) + prompt = artifact_dir / "prompt.md" + prompt.write_text("Original discovery input") + run_workbench( + state, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact_dir), + "--attempt", + "1", + ) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + f"UPDATE deep_scan_runs SET {field} = ?, updated_at = ?", + (value, "2000-01-01T00:00:00Z"), + ) + if operation == "handoff": + connection.execute( + "UPDATE scans SET deep_scan_owner_thread_id = NULL, recipe_json = '{}'" + ) + connection.execute("UPDATE workspaces SET thread_id = NULL") + before = snapshot(state) + command = "claim-deep-scan-coordinator" if operation == "claim" else "begin-deep-scan" + result = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + *(["--model", "observer-model"] if operation != "claim" else []), + check=False, + ) + assert result["returncode"] != 0 + assert "unsupported" in str(result["stderr"]).lower() + assert snapshot(state) == before + if operation != "handoff": + observed = run_workbench( + state, "get-deep-scan", "--scan-id", scan_id, "--thread-id", "fixture-thread" + )["deepScan"] + assert ( + observed["workflowVersion" if field == "workflow_version" else "schemaVersion"] == value + ) + assert snapshot(state) == before + + +@pytest.mark.parametrize("original", [None, "Original discovery context"]) +def test_reader_honors_original_context_when_present(tmp_path: Path, original: str | None) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "discovery_user_context" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN discovery_user_context TEXT") + connection.execute("UPDATE deep_scan_runs SET discovery_user_context = ?", (original,)) + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["userContext"] == original + + +def test_unsupported_new_workflow_does_not_claim_registered_scan(tmp_path: Path) -> None: + state = tmp_path / "state" + target = tmp_path / "target" + target.mkdir() + scan_dir = tmp_path / "scan" + scan_dir.mkdir(mode=0o700) + registered = run_workbench( + state, + "register-cli-scan", + "--scan-dir", + str(scan_dir), + "--repository", + str(target), + "--registration-json-stdin", + input_text=json.dumps( + { + "recipe": { + "config": {}, + "mode": "deep", + "repository": str(target), + "target": {"kind": "repository", "paths": []}, + } + } + ), + ) + before = snapshot(state) + rejected = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + str(registered["scanId"]), + "--thread-id", + "fixture-thread", + "--workflow-version", + "future/v99", + check=False, + ) + assert rejected["returncode"] != 0 + assert "unsupported" in str(rejected["stderr"]).lower() + assert snapshot(state) == before diff --git a/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py new file mode 100644 index 000000000..6e1cd3a02 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_finalization_compatibility.py @@ -0,0 +1,123 @@ +"""Selected finalization survives ownership recovery without restarting discovery.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +@pytest.mark.parametrize("legacy_version", [None, "deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_new_workflow_default_preserves_existing_run_version( + tmp_path: Path, legacy_version: str | None +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + created = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + *(["--workflow-version", legacy_version] if legacy_version else []), + )["deepScan"] + expected_version = legacy_version or "deep-security-scan/v2" + assert created["workflowVersion"] == expected_version + resumed = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + created["scanId"], + "--thread-id", + "fixture-thread", + )["deepScan"] + assert resumed["workflowVersion"] == expected_version + assert resumed["createdAt"] == created["createdAt"] + + +def selected_scan(tmp_path: Path, version: int) -> tuple[Path, str, dict[str, object]]: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + run = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + )["deepScan"] + selected = { + "version": version, + "resultPath": None, + "resultSha256": None, + "terminalReason": "capped", + "omittedWorkerIds": [], + "selectedAt": "2000-01-01T00:00:00Z", + } + with sqlite3.connect(state / "workbench.sqlite3") as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")} + if "finalization_input_json" not in columns: + connection.execute("ALTER TABLE deep_scan_runs ADD COLUMN finalization_input_json TEXT") + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, phase = 'reducing', " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z'", + (json.dumps(selected),), + ) + return state, str(run["scanId"]), selected + + +def test_claim_preserves_selected_finalization_without_discovery_recovery(tmp_path: Path) -> None: + state, scan_id, selected = selected_scan(tmp_path, 1) + claimed = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert claimed["finalizationInput"] == selected + assert claimed["phase"] == "reducing" + assert claimed["coordinatorGeneration"] == 2 + assert claimed["dispatchedCount"] == 0 + assert claimed["createdAt"] == "2000-01-01T00:00:00Z" + + +@pytest.mark.parametrize("command", ["begin-deep-scan", "claim-deep-scan-coordinator"]) +def test_unsupported_selection_rejects_without_mutation(tmp_path: Path, command: str) -> None: + state, scan_id, selected = selected_scan(tmp_path, 99) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + before = "\n".join(connection.iterdump()) + rejected = run_workbench( + state, + command, + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + check=False, + ) + assert rejected["returncode"] != 0 + assert "unsupported" in str(rejected["stderr"]).lower() + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["finalizationInput"] == selected diff --git a/plugins/codex-security/tests/test_deep_scan_persistence.py b/plugins/codex-security/tests/test_deep_scan_persistence.py new file mode 100644 index 000000000..0f2f0e67e --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_persistence.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +import uuid +from pathlib import Path + +import pytest +from test_workbench_deep_scan import ( + begin_target_scan, + commit_reducer, + dispatch_discovery_worker, + upsert_worker, + worker_paths, +) +from workbench_test_support import run_workbench + + +def test_state_snapshot_does_not_mix_concurrent_acceptance( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + state_dir, codex_home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + initial = begin_target_scan(state_dir, codex_home, target, tmp_path / "scans")["deepScan"] + scan_id = initial["scanId"] + worker_id, _, _, _ = dispatch_discovery_worker( + state_dir, + codex_home, + scan_id=scan_id, + scan_dir=Path(initial["scanDir"]), + name="discovery-1", + succeed=False, + ) + database = state_dir / "workbench.sqlite3" + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr(deep_scan, "require_scan", workbench_api["require_scan"]) + original = deep_scan.require_deep_scan_run + + def accept_after_read(connection, requested_scan_id): + run = original(connection, requested_scan_id) + with sqlite3.connect(database) as writer: + writer.execute( + "UPDATE deep_scan_runs SET completion_sequence = 1 WHERE scan_id = ?", (scan_id,) + ) + writer.execute( + "UPDATE deep_scan_workers SET status = 'succeeded', completion_sequence = 1 " + "WHERE id = ?", + (worker_id,), + ) + return run + + monkeypatch.setattr(deep_scan, "require_deep_scan_run", accept_after_read) + with sqlite3.connect(database) as reader: + reader.row_factory = sqlite3.Row + snapshot = deep_scan.deep_scan_state(reader, scan_id) + assert snapshot["completionSequence"] == 0 + assert snapshot["workers"][0]["status"] == "running" + assert not reader.in_transaction + + +def test_state_snapshot_preserves_its_callers_transaction( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + target = tmp_path / "target" + target.mkdir() + initial = begin_target_scan(tmp_path / "state", tmp_path / "codex", target, tmp_path / "scans") + scan_id = initial["deepScan"]["scanId"] + deep_scan = sys.modules["deep_scan_workbench"] + monkeypatch.setattr(deep_scan, "require_scan", workbench_api["require_scan"]) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "UPDATE deep_scan_runs SET consecutive_errors = 2 WHERE scan_id = ?", (scan_id,) + ) + snapshot = deep_scan.deep_scan_state(connection, scan_id) + assert snapshot["consecutiveErrors"] == 2 + assert connection.in_transaction + connection.rollback() + assert deep_scan.deep_scan_state(connection, scan_id)["consecutiveErrors"] == 0 + + +def test_replaced_attempts_retain_observed_sessions_and_accepted_result(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery-1") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + ) + upsert_worker(state, home, **mutation, status="running", attempt=1, thread_id="old-session") + upsert_worker( + state, + home, + **mutation, + status="running", + attempt=1, + thread_id="old-session", + error="Artifact validation failed", + ) + upsert_worker(state, home, **mutation, status="running", attempt=2, thread_id="new-session") + result.write_text('{"findings": []}\n') + accepted = upsert_worker( + state, + home, + **mutation, + status="succeeded", + attempt=2, + thread_id="new-session", + result_path=result, + )["deepScan"] + attempts = accepted["attempts"] + assert [(item["attempt"], item["status"]) for item in attempts] == [ + (1, "failed"), + (2, "succeeded"), + ] + assert [item["sdkThreadId"] for item in accepted["attemptSessions"]] == [ + "old-session", + "new-session", + ] + assert attempts[0]["error"] == "Artifact validation failed" + assert attempts[0]["completedAt"] is not None + assert attempts[1]["acceptedResultSha256"] + assert Path(attempts[1]["acceptedResultPath"]).read_text() == result.read_text() + result.unlink() + replayed = upsert_worker( + state, + home, + **mutation, + status="succeeded", + attempt=2, + thread_id="new-session", + result_path=result, + )["deepScan"] + assert replayed == accepted + + +def test_merge_replay_returns_original_operation_after_later_work(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + inputs = [ + dispatch_discovery_worker( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name=f"discovery-{index}", + )[0] + for index in range(2) + ] + committed = commit_reducer( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="dedup-1", + input_worker_ids=inputs, + new_findings_count=0, + ) + reducer = next(worker for worker in committed["workers"] if worker["kind"] == "dedup") + frozen = committed["committedMerge"]["resultManifestPath"] + assert Path(frozen).is_file() + Path(reducer["resultManifestPath"]).unlink() + assert [item["discoveryWorkerId"] for item in committed["dedupInputs"]] == inputs + assert all(item["attempt"] == 1 for item in committed["dedupInputs"]) + assert all("/checkpoints/" in item["resultManifestPath"] for item in committed["dedupInputs"]) + later = dispatch_discovery_worker( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="discovery-2", + )[0] + second_commit = commit_reducer( + state, + home, + scan_id=scan_id, + scan_dir=scan_dir, + name="dedup-2", + input_worker_ids=[later], + new_findings_count=1, + ) + second_claim = second_commit["mergeClaims"][-1] + assert second_claim["previousWorkerId"] == reducer["id"] + assert second_claim["previousResultPath"] == frozen + assert ( + second_claim["previousResultSha256"] == committed["committedMerge"]["resultManifestSha256"] + ) + replay = run_workbench( + state, + "commit-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + reducer["id"], + "--result-manifest-path", + str(scan_dir / "different.json"), + "--new-findings-count", + "99", + environment={"CODEX_HOME": str(home)}, + )["deepScan"] + assert replay["committedMerge"] == committed["committedMerge"] + assert replay["completionSequence"] == second_commit["completionSequence"] + assert replay["noNewStreak"] == second_commit["noNewStreak"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + receipt = json.loads( + connection.execute( + "SELECT receipt_json FROM deep_scan_merge_claims WHERE worker_id = ?", + (reducer["id"],), + ).fetchone()[0] + ) + assert receipt == committed["committedMerge"] + + +def test_native_usage_keeps_replaced_failed_canceled_attempts_and_descendants( + tmp_path: Path, workbench_api, monkeypatch: pytest.MonkeyPatch +) -> None: + from datetime import datetime, timedelta + + from test_workbench_scan_usage import _counts, _event, _rollout, _state_graph, _token_event + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, _ = worker_paths(Path(run["scanDir"]), "discovery-1") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + ) + upsert_worker(state, home, **mutation, status="running", attempt=1, thread_id="old") + upsert_worker(state, home, **mutation, status="running", attempt=2, thread_id="old") + upsert_worker( + state, + home, + **mutation, + status="running", + attempt=2, + thread_id="old", + error="fixture failure", + ) + upsert_worker(state, home, **mutation, status="running", attempt=3, thread_id="replacement") + terminal = upsert_worker( + state, home, **mutation, status="canceled", attempt=3, thread_id="replacement" + )["deepScan"] + assert [item["status"] for item in terminal["attempts"]] == ["replaced", "failed", "canceled"] + environment = { + "CODEX_HOME": str(home), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + for key, value in environment.items(): + monkeypatch.setenv(key, value) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.row_factory = sqlite3.Row + scan = connection.execute("SELECT * FROM scans WHERE id = ?", (run["scanId"],)).fetchone() + timestamp = datetime.fromisoformat(scan["started_at"]) + timedelta(microseconds=1) + context = _event( + timestamp, "turn_context", {"turn_id": "fixture-turn", "model": "gpt-5.6-sol"} + ) + old = _rollout(tmp_path, "old", [context]) + _state_graph( + environment, + { + "thread-deep-scan": _rollout(tmp_path, "thread-deep-scan", []), + "old": old, + "replacement": _rollout( + tmp_path, "replacement", [context, _token_event(timestamp, 30, 0)] + ), + "child": _rollout( + tmp_path, + "child", + [context, _token_event(timestamp, 5, 0)], + parent_thread_id="old", + ), + "unrelated": _rollout( + tmp_path, + "unrelated", + [context, _token_event(timestamp, 900, 0)], + parent_thread_id="thread-deep-scan", + ), + }, + [("old", "child"), ("thread-deep-scan", "unrelated")], + ) + reader = sys.modules["workbench_scan_usage"] + pending = reader.collect_scan_usage(connection, scan) + assert pending["inputTokens"] == 35 + assert pending["missingThreadCount"] == 2 + old.write_text(old.read_text() + json.dumps(_token_event(timestamp, 20, 0)) + "\n") + measured = reader.collect_scan_usage(connection, scan) + assert measured["inputTokens"] == 55 + assert measured["threadCount"] == 3 + assert measured["missingThreadCount"] == 1 + assert measured["coverage"] == "partial" # Original shared parent turn was unavailable. + assert measured["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(55, 0, 0)}] + + +def test_claim_replay_preserves_original_inputs_after_concurrent_discovery(tmp_path: Path) -> None: + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + scan_id, scan_dir = run["scanId"], Path(run["scanDir"]) + inputs = [ + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name=f"discovery-{index}" + )[0] + for index in range(2) + ] + prompt, artifacts, _ = worker_paths(scan_dir, "reducer") + args = [ + "claim-deep-scan-dedup", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifacts), + ] + for worker in inputs: + args.extend(["--input-worker-id", worker]) + claimed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + dispatch_discovery_worker( + state, home, scan_id=scan_id, scan_dir=scan_dir, name="concurrent-discovery" + ) + replayed = run_workbench(state, *args, environment={"CODEX_HOME": str(home)}) + assert replayed["deepScan"]["mergeClaims"] == claimed["deepScan"]["mergeClaims"] + assert replayed["deepScan"]["dedupInputs"] == claimed["deepScan"]["dedupInputs"] + assert ( + replayed["deepScan"]["completionSequence"] == claimed["deepScan"]["completionSequence"] + 1 + ) + + +def test_acceptance_reuses_authoritative_checkpoint_without_rewriting(tmp_path: Path) -> None: + import hashlib + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=1, + ) + upsert_worker(state, home, **mutation, status="running") + draft = {"scanId": run["scanId"], "findings": [], "coverage": {}} + content = json.dumps(draft, indent=2).encode() + b"\n" + checkpoint = artifacts / "checkpoints" / f"{hashlib.sha256(content).hexdigest()}.json" + checkpoint.parent.mkdir() + checkpoint.write_bytes(content) + (artifacts / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + result.write_text(json.dumps({**draft, "handoffClaimToken": "synthetic-claim"})) + accepted = upsert_worker(state, home, **mutation, status="succeeded", result_path=result)[ + "deepScan" + ] + assert accepted["attempts"][0]["acceptedResultPath"] == str(checkpoint) + assert accepted["attempts"][0]["acceptedResultSha256"] == hashlib.sha256(content).hexdigest() + result.unlink() + upsert_worker(state, home, **mutation, status="succeeded", result_path=result) + assert list(checkpoint.parent.iterdir()) == [checkpoint] + assert checkpoint.read_bytes() == content + assert not (artifacts / "accepted").exists() + + +def test_acceptance_rejects_mutable_result_behind_checkpoint_head(tmp_path: Path) -> None: + import hashlib + + state, home, target = tmp_path / "state", tmp_path / "codex", tmp_path / "target" + target.mkdir() + run = begin_target_scan(state, home, target, tmp_path / "scans")["deepScan"] + worker_id = str(uuid.uuid4()) + prompt, artifacts, result = worker_paths(Path(run["scanDir"]), "discovery") + mutation = dict( + scan_id=run["scanId"], + worker_id=worker_id, + kind="discovery", + prompt_path=prompt, + artifact_dir=artifacts, + attempt=1, + ) + upsert_worker(state, home, **mutation, status="running") + draft = {"scanId": run["scanId"], "findings": [], "coverage": {"deferred": ["unresolved"]}} + content = json.dumps(draft).encode() + checkpoint = artifacts / "checkpoints" / f"{hashlib.sha256(content).hexdigest()}.json" + checkpoint.parent.mkdir() + checkpoint.write_bytes(content) + (artifacts / "checkpoint-head.json").write_text(json.dumps({"checkpoint": checkpoint.name})) + result.write_text(json.dumps({**draft, "coverage": {}})) + with pytest.raises(subprocess.CalledProcessError) as failure: + upsert_worker(state, home, **mutation, status="succeeded", result_path=result) + assert "does not match its current checkpoint head" in failure.value.stderr + assert checkpoint.read_bytes() == content diff --git a/plugins/codex-security/tests/test_deep_scan_publication_authority.py b/plugins/codex-security/tests/test_deep_scan_publication_authority.py new file mode 100644 index 000000000..b4ff3a0bd --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_publication_authority.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import copy +import json +import uuid +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def stage_publication(scan, *, generation, result_path, title): + draft_dir = scan.scan_dir / "drafts" + draft_dir.mkdir(exist_ok=True) + draft_path = draft_dir / f"{uuid.uuid4()}.json" + checkpoint_path = draft_dir / f"{uuid.uuid4()}.checkpoint.json" + findings = copy.deepcopy(scan.findings) + findings[0]["title"] = title + draft = { + "manifest": json.loads((scan.scan_dir / "scan-manifest.json").read_text()), + "findings": {"findings": findings}, + "coverage": scan.coverage, + } + if generation is not None: + draft["deepScanPublication"] = { + "coordinatorGeneration": generation, + "resultPath": str(result_path), + } + draft_path.write_text(json.dumps(draft)) + checkpoint_path.write_text( + json.dumps({"scanId": scan.scan_id, "findings": findings, "coverage": scan.coverage}) + ) + return Namespace( + scan_id=scan.scan_id, + claim_token=None, + draft_path=str(draft_path), + checkpoint_path=str(checkpoint_path), + expected_draft_digest=None, + ) + + +@pytest.mark.parametrize("stale", ["generation", "aggregate", "unfenced"]) +def test_stale_coordinator_cannot_replace_newer_canonical_publication( + workbench_api, workbench_db, publication_scan, stale +): + scan = publication_scan() + old_result = add_worker(workbench_db, scan) + new_result = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + for result, completed_at in ((old_result, "2026-01-01"), (new_result, "2026-01-02")): + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none', completed_at = ? " + "WHERE result_manifest_path = ?", + (completed_at, str(result)), + ) + current = stage_publication( + scan, generation=3, result_path=new_result, title="Current accepted aggregate" + ) + workbench_api["write_scan_draft"](workbench_db, current) + saved = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*.json") + if "drafts" not in path.parts + } + old = stage_publication( + scan, + generation=None if stale == "unfenced" else 2 if stale == "generation" else 3, + result_path=old_result if stale == "aggregate" else new_result, + title="Superseded aggregate", + ) + + with pytest.raises(SystemExit, match="coordinator|aggregate"): + workbench_api["write_scan_draft"](workbench_db, old) + + assert { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*.json") + if "drafts" not in path.parts + } == saved diff --git a/plugins/codex-security/tests/test_deep_scan_publication_replay.py b/plugins/codex-security/tests/test_deep_scan_publication_replay.py new file mode 100644 index 000000000..e7c361569 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_publication_replay.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import saved_selection + +_CRASH_PUBLICATION = """ +import json, os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="publication_crash_test") +args = Namespace(**json.loads(sys.argv[3])) +boundary = sys.argv[4] + +class CrashConnection(sqlite3.Connection): + def commit(self): + completing = self.execute( + "SELECT status FROM scans WHERE id = ?", (args.scan_id,) + ).fetchone()[0] == "complete" + if completing and boundary == "sqlite-before": + os._exit(72) + super().commit() + if completing and boundary == "sqlite-after": + os._exit(73) + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +if boundary.startswith("sqlite-"): + api["complete_scan"](connection, Namespace( + scan_id=args.scan_id, claim_token=None, cost_json=None + )) +else: + saved = api["saved_results"] + original_write = saved.write_scan_local_bytes + def crash_after_write(root, relative, contents): + original_write(root, relative, contents) + if relative == boundary: + os._exit(71) + saved.write_scan_local_bytes = crash_after_write + api["write_scan_draft"](connection, args) +raise AssertionError("publication never reached the requested crash boundary") +""" + + +@pytest.mark.parametrize( + "boundary", + ["findings.json", "coverage.json", "scan-manifest.json", "sqlite-before", "sqlite-after"], +) +@pytest.mark.parametrize("selection_reason", [None, "saturated", "capped"]) +def test_publication_crash_replays_selected_input_without_stale_overwrite( + workbench_api, workbench_db, publication_scan, tmp_path, boundary, selection_reason +): + scan = publication_scan() + if selection_reason is None: + result_path = add_worker(workbench_db, scan) + else: + _, result_path, _ = accept_reducer(workbench_db, scan) + saved_selection(workbench_db, scan, result_path, reason=selection_reason) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET terminal_reason = ? WHERE scan_id = ?", + (selection_reason, scan.scan_id), + ) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = 3 WHERE scan_id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE scan_id = ?", + (scan.scan_id,), + ) + current = stage_publication( + scan, generation=3, result_path=result_path, title="Selected aggregate" + ) + stale = stage_publication( + scan, generation=2, result_path=result_path, title="Obsolete coordinator draft" + ) + database_path = tmp_path / "publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + if boundary.startswith("sqlite-"): + workbench_api["write_scan_draft"](connection, current) + + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_PUBLICATION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + json.dumps(vars(current)), + boundary, + ], + capture_output=True, + text=True, + ) + assert child.returncode == {"sqlite-before": 72, "sqlite-after": 73}.get(boundary, 71), ( + child.stdout, + child.stderr, + ) + interrupted = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.parts + } + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = connection.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["status"] == ("complete" if boundary == "sqlite-after" else "running") + assert bool(row["seal_manifest_digest"]) == (boundary == "sqlite-after") + run_before = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + workers_before = [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] + + with pytest.raises(SystemExit, match="coordinator|stopped"): + workbench_api["write_scan_draft"](connection, stale) + assert all(path.read_bytes() == contents for path, contents in interrupted.items()) + + if not boundary.startswith("sqlite-"): + workbench_api["write_scan_draft"](connection, current) + completed = workbench_api["complete_scan"]( + connection, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + )["scan"] + assert completed["progress"]["status"] == "complete" + assert completed["findingCount"] == 1 + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run_before + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers_before + assert connection.execute("SELECT COUNT(*) FROM finding_occurrences").fetchone()[0] == 1 + published = { + path: path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.parts + } + workbench_api["complete_scan"]( + connection, Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + ) + assert all(path.read_bytes() == contents for path, contents in published.items()) + if boundary.startswith("sqlite-"): + assert published == interrupted + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["title"] == "Selected aggregate" diff --git a/plugins/codex-security/tests/test_deep_scan_recovery_settings.py b/plugins/codex-security/tests/test_deep_scan_recovery_settings.py new file mode 100644 index 000000000..37976c3e1 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_recovery_settings.py @@ -0,0 +1,131 @@ +"""Original discovery input and observation remain stable across reconstruction.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +@pytest.mark.parametrize("original_context", [None, "Audit the original parser"]) +def test_reconstruction_preserves_discovery_input_settings_and_deadline( + tmp_path: Path, + original_context: str | None, +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + codex_home = tmp_path / "home" + config = codex_home / "codex-security" / "config.toml" + config.parent.mkdir(parents=True) + config.write_text("[deep_scan]\nworkers = 2\nmax_time_hours = 2.5\n") + environment = {"CODEX_HOME": str(codex_home)} + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--model", + "original-model", + "--reasoning-effort", + "high", + *(["--user-context-stdin"] if original_context is not None else []), + input_text=original_context, + environment=environment, + )["deepScan"] + scan_id = str(begun["scanId"]) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + connection.execute("UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00Z'") + before = "\n".join(connection.iterdump()) + config.write_text("[deep_scan]\nworkers = 8\nmax_time_hours = 12\n") + joined = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + "--model", + "observer-model", + "--reasoning-effort", + "low", + environment=environment, + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert "\n".join(connection.iterdump()) == before + assert connection.execute("SELECT model, reasoning_effort FROM scans").fetchone() == ( + "original-model", + "high", + ) + recovered = run_workbench( + state, + "claim-deep-scan-coordinator", + "--scan-id", + scan_id, + "--thread-id", + "fixture-thread", + environment=environment, + )["deepScan"] + for run in (joined, recovered): + assert run["model"] == "original-model" + assert run["reasoningEffort"] == "high" + assert run["userContext"] == original_context + assert run["createdAt"] == begun["createdAt"] + assert run["config"] == begun["config"] + assert run["workflowVersion"] == begun["workflowVersion"] + + +@pytest.mark.parametrize("workflow_version", ["deep-security-scan/v1", "deep-scan-mcp/v1"]) +def test_supported_old_run_snapshots_context_on_upgrade( + tmp_path: Path, workflow_version: str +) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + begun = run_workbench( + state, + "begin-deep-scan", + "--workflow-version", + workflow_version, + "--thread-id", + "fixture-thread", + "--target-path", + str(target), + "--scan-root", + str(tmp_path / "scans"), + "--user-context", + "Legacy context", + )["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("ALTER TABLE deep_scan_runs DROP COLUMN discovery_user_context") + connection.execute("DELETE FROM schema_migrations WHERE version = 44") + upgraded = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert upgraded["workflowVersion"] == workflow_version + assert upgraded["userContext"] == "Legacy context" + assert upgraded["config"] == begun["config"] + assert upgraded["createdAt"] == begun["createdAt"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute("UPDATE scans SET user_context = 'Later discussion'") + observed = run_workbench( + state, + "get-deep-scan", + "--scan-id", + str(begun["scanId"]), + "--thread-id", + "fixture-thread", + )["deepScan"] + assert observed["userContext"] == "Legacy context" diff --git a/plugins/codex-security/tests/test_deep_scan_settings_creation.py b/plugins/codex-security/tests/test_deep_scan_settings_creation.py new file mode 100644 index 000000000..4ff1a77b4 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_settings_creation.py @@ -0,0 +1,298 @@ +"""Original settings survive real creation/claim process death and competing starts.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest +from workbench_test_support import ( + SCRIPT, + create_saved_workspace, + run_workbench, + start_delivered_scan, +) + +# Run the real workbench, pausing only at a requested transaction boundary. +# The parse hook also runs against the parent source, where the settings input +# was ignored, so its regression fails on missing durable bytes. +BOUNDARY_PROCESS = """ +import json, runpy, sys +from pathlib import Path +boundary = sys.argv.pop(1) +script = sys.argv.pop(1) +main = runpy.run_path(script)['main'] +namespace = main.__globals__ +deep = namespace['deep_scan'] +parse = namespace['parse_args'] +def parse_with_settings(*args, **kwargs): + result = parse(*args, **kwargs) + payload = json.loads(sys.stdin.readline()) + result.execution_settings = payload['executionSettings'] + result.user_context = payload.get('userContext') + result.user_context_stdin = False + return result +namespace['parse_args'] = parse_with_settings +def pause(scan): + print(json.dumps({'scanId': scan['id'], 'scanDir': scan['scan_dir']}), flush=True) + sys.stdin.readline() +write = deep.write_scan_local_bytes +def write_settings(scan_dir, relative_path, payload, **kwargs): + if boundary == 'before-write' and relative_path.endswith('/execution-settings.json'): + pause({'id': None, 'scan_dir': str(scan_dir)}) + return write(scan_dir, relative_path, payload, **kwargs) +deep.write_scan_local_bytes = write_settings +ensure = deep.ensure_deep_scan_run +def ensure_run(connection, scan, *args): + result = ensure(connection, scan, *args) + if boundary == 'before-commit': + pause(scan) + return result +deep.ensure_deep_scan_run = ensure_run +begin = deep.begin_deep_scan +def begin_run(connection, args): + result = begin(connection, args) + if boundary == 'after-commit': + pause({'id': result['deepScan']['scanId'], 'scan_dir': result['deepScan']['scanDir']}) + return result +deep.begin_deep_scan = begin_run +claim = deep.claim_deep_scan_coordinator +def claim_run(connection, args): + result = claim(connection, args) + if boundary == 'after-claim': + pause({'id': result['deepScan']['scanId'], 'scan_dir': result['deepScan']['scanDir']}) + return result +deep.claim_deep_scan_coordinator = claim_run +main() +""" + + +def settings(root: Path, name: str) -> dict[str, object]: + return { + "codexPath": str(root / name / "codex"), + "codexHome": str(root / name / "home"), + "model": f"{name}-model", + "reasoningEffort": "high", + "reasoningSummary": "concise", + "parentSandbox": {"filesystemDenies": [str(root / name / "denied")]}, + } + + +def start_process( + root: Path, + selected: dict[str, object] | None, + *args: str, + boundary: str = "none", +) -> subprocess.Popen[str]: + process = subprocess.Popen( + [sys.executable, "-c", BOUNDARY_PROCESS, boundary, str(SCRIPT), *args], + env={ + **os.environ, + "CODEX_SECURITY_STATE_DIR": str(root / "state"), + "CODEX_HOME": str(root / "current-home"), + }, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdin is not None + process.stdin.write(json.dumps({"executionSettings": selected}) + "\n") + process.stdin.flush() + return process + + +def finish(process: subprocess.Popen[str]) -> dict[str, object]: + stdout, stderr = process.communicate(timeout=30) + assert process.returncode == 0, stderr + return json.loads(stdout) + + +def begin_args(root: Path, *, name: str = "target") -> list[str]: + (root / name).mkdir(exist_ok=True) + return [ + "begin-deep-scan", + "--thread-id", + "original-thread", + "--target-path", + str(root / name), + "--scan-root", + str(root / "scans"), + ] + + +def snapshot(scan_dir: str) -> Path: + return Path(scan_dir) / "artifacts" / "deep_discovery" / "execution-settings.json" + + +@pytest.mark.parametrize( + "boundary", ["before-write", "before-commit", "after-commit", "after-claim"] +) +def test_process_death_keeps_original_settings_or_no_recoverable_run( + tmp_path: Path, boundary: str +) -> None: + original = settings(tmp_path, "original") + args = begin_args(tmp_path) + if boundary == "after-claim": + begun = finish(start_process(tmp_path, original, *args))["deepScan"] + args = [ + "claim-deep-scan-coordinator", + "--scan-id", + begun["scanId"], + "--thread-id", + "original-thread", + ] + process = start_process(tmp_path, original, *args, boundary=boundary) + try: + assert process.stdout is not None + paused = json.loads(process.stdout.readline()) + path = snapshot(paused["scanDir"]) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + rows = connection.execute("SELECT scan_id FROM deep_scan_runs").fetchall() + if boundary in {"before-write", "before-commit"}: + assert rows == [], "uncommitted settings must not leave a recoverable run" + else: + assert rows == [(paused["scanId"],)] + if boundary != "before-write": + assert json.loads(path.read_bytes())["settings"] == original + finally: + process.kill() + process.communicate(timeout=30) + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + remaining = connection.execute("SELECT scan_id FROM deep_scan_runs").fetchall() + if boundary in {"before-write", "before-commit"}: + assert remaining == [] + return + before = path.read_bytes() + with sqlite3.connect(tmp_path / "state" / "workbench.sqlite3") as connection: + connection.execute("UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00Z'") + run_workbench( + tmp_path / "state", + "claim-deep-scan-coordinator", + "--scan-id", + paused["scanId"], + "--thread-id", + "original-thread", + ) + joined = finish(start_process(tmp_path, settings(tmp_path, "later"), *begin_args(tmp_path))) + assert joined["startDisposition"] == "joined" + assert path.read_bytes() == before + + +def test_competing_creation_and_observer_cannot_replace_original_settings(tmp_path: Path) -> None: + first = start_process( + tmp_path, settings(tmp_path, "original"), *begin_args(tmp_path), boundary="before-commit" + ) + second = None + try: + assert first.stdout is not None + paused = json.loads(first.stdout.readline()) + path = snapshot(paused["scanDir"]) + before = path.read_bytes() + second = start_process(tmp_path, settings(tmp_path, "later"), *begin_args(tmp_path)) + # Release the creation lock with the original process still alive. The + # contender must observe the committed run, not rewrite the settings. + created = finish(first) + joined = finish(second) + assert created["startDisposition"] == "created" + assert joined["startDisposition"] == "joined" + assert joined["deepScan"]["scanId"] == paused["scanId"] + assert path.read_bytes() == before + observer = finish(start_process(tmp_path, None, *begin_args(tmp_path))) + assert observer["startDisposition"] == "joined" + assert path.read_bytes() == before + other = finish( + start_process( + tmp_path, settings(tmp_path, "other"), *begin_args(tmp_path, name="other-target") + ) + ) + assert json.loads(snapshot(other["deepScan"]["scanDir"]).read_bytes())[ + "settings" + ] == settings(tmp_path, "other") + finally: + for process in (first, second): + if process is not None and process.poll() is None: + process.kill() + process.communicate(timeout=30) + + +def test_internal_settings_input_preserves_public_user_context_and_shape(tmp_path: Path) -> None: + context = "Review the parser.\nKeep this second line." + process = subprocess.run( + [ + sys.executable, + "-c", + ( + "import runpy, sys; script = sys.argv.pop(1); " + "runpy.run_path(script)['main'](with_execution_settings=True)" + ), + str(SCRIPT), + *begin_args(tmp_path), + ], + env={**os.environ, "CODEX_SECURITY_STATE_DIR": str(tmp_path / "state")}, + input=json.dumps( + {"executionSettings": settings(tmp_path, "original"), "userContext": context} + ), + capture_output=True, + text=True, + timeout=30, + ) + assert process.returncode == 0, process.stderr + result = json.loads(process.stdout) + assert result["deepScan"]["userContext"] == context + assert "executionSettings" not in result["deepScan"] + assert json.loads(snapshot(result["deepScan"]["scanDir"]).read_bytes())["settings"] == settings( + tmp_path, "original" + ) + + +def test_managed_creation_retry_preserves_saved_settings_before_commit(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + state = tmp_path / "state" + workspace = create_saved_workspace(state, target, thread_id="original-thread", mode="deep") + started = start_delivered_scan( + state, + "--workspace-id", + workspace["id"], + "--scan-root", + str(tmp_path / "scans"), + "--model", + "original-model", + "--reasoning-effort", + "high", + ) + scan = started["results"] + args = ["begin-deep-scan", "--scan-id", scan["scanId"], "--thread-id", "original-thread"] + original = settings(tmp_path, "original") + process = start_process(tmp_path, original, *args, boundary="before-commit") + try: + assert process.stdout is not None + paused = json.loads(process.stdout.readline()) + path = snapshot(paused["scanDir"]) + before = path.read_bytes() + finally: + process.kill() + process.communicate(timeout=30) + with sqlite3.connect(state / "workbench.sqlite3") as connection: + assert connection.execute("SELECT COUNT(*) FROM deep_scan_runs").fetchone() == (0,) + recovered = finish( + start_process( + tmp_path, + settings(tmp_path, "later"), + *args, + "--model", + "later-model", + "--reasoning-effort", + "low", + ) + ) + assert recovered["startDisposition"] == "created" + assert path.read_bytes() == before + assert json.loads(before)["settings"] == original + assert recovered["deepScan"]["model"] == "original-model" + assert recovered["deepScan"]["reasoningEffort"] == "high" diff --git a/plugins/codex-security/tests/test_deep_scan_successful_publication.py b/plugins/codex-security/tests/test_deep_scan_successful_publication.py index 8a144ece4..b9c9c9e14 100644 --- a/plugins/codex-security/tests/test_deep_scan_successful_publication.py +++ b/plugins/codex-security/tests/test_deep_scan_successful_publication.py @@ -72,7 +72,7 @@ def create(*, mode="deep", scope="."): "INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, " "status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, " "manifest_path, terminal_reason, created_at, updated_at, completed_at) " - "VALUES (?, 1, 'publication-test', 'succeeded', 'terminal', 1, 0, 1, 1, " + "VALUES (?, 1, 'deep-security-scan/v1', 'succeeded', 'terminal', 1, 0, 1, 1, " "?, 'saturated', ?, ?, ?)", ( scan_id, @@ -173,6 +173,38 @@ def assert_published_aggregate(scan): assert (scan.scan_dir / "report.md").is_file() +def test_deep_publication_renders_each_source_remediation( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + finding = scan.findings[0] + first = copy.deepcopy(finding) + first.pop("provenance") + first["remediation"] = "Check the destination before writing the archive entry." + first["remediationTests"] = ["Reject an archive entry outside the destination."] + second = copy.deepcopy(first) + second["remediation"] = "Reject symbolic links before opening the destination." + second["remediationTests"] = ["Reject a symbolic link inside the destination."] + second["preventiveControls"] = ["Use a directory-relative file handle."] + finding["remediation"] = first["remediation"] + finding["remediationTests"] = first["remediationTests"] + finding["provenance"]["sourceFindings"] = [ + {"id": "review-1:0", "finding": first}, + {"id": "review-2:0", "finding": second}, + ] + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": scan.findings})) + + complete(workbench_api, workbench_db, scan) + + assert_published_aggregate(scan) + report = (scan.scan_dir / "report.md").read_text() + for source in (first, second): + assert report.count(source["remediation"]) == 1 + for test in source["remediationTests"]: + assert report.count(test) == 1 + assert "Use a directory-relative file handle." in report + + @pytest.mark.parametrize("scope", [".", "subdir"], ids=["repository", "scoped"]) def test_deep_publication_keeps_configured_scope_without_worker_observations( workbench_api, workbench_db, publication_scan, scope @@ -460,3 +492,32 @@ def test_standard_publication_preserves_deliberately_partial_coverage( complete(workbench_api, workbench_db, scan) assert_published_aggregate(scan) + + +def test_deep_publication_write_failure_keeps_original_terminal_cause( + workbench_api, workbench_db, publication_scan, monkeypatch +): + scan = publication_scan() + finalizer_globals = workbench_api["_write_prepared_scan_finalization"].__globals__ + write_bytes = finalizer_globals["write_scan_local_bytes"] + + def fail_report(scan_dir, relative_path, payload, **kwargs): + if relative_path == "report.md": + raise finalizer_globals["ContractError"]("Synthetic report write interruption") + return write_bytes(scan_dir, relative_path, payload, **kwargs) + + with monkeypatch.context() as patch: + patch.setitem(finalizer_globals, "write_scan_local_bytes", fail_report) + with pytest.raises(SystemExit, match="Synthetic report write interruption"): + complete(workbench_api, workbench_db, scan) + + assert ( + workbench_db.execute("SELECT status FROM scans WHERE id = ?", (scan.scan_id,)).fetchone()[0] + == "running" + ) + run = workbench_db.execute( + "SELECT status, terminal_reason FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + assert tuple(run) == ("succeeded", "saturated") + assert complete(workbench_api, workbench_db, scan)["progress"]["status"] == "complete" + assert_published_aggregate(scan) diff --git a/plugins/codex-security/tests/test_deep_scan_usage_owner.py b/plugins/codex-security/tests/test_deep_scan_usage_owner.py new file mode 100644 index 000000000..9b5cb237c --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_usage_owner.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +from test_workbench_scan_usage import _event, _rollout, _state_graph +from workbench_test_support import run_workbench + + +def test_original_usage_turn_survives_join_and_coordinator_recovery(tmp_path: Path) -> None: + state = tmp_path / "state" + environment = { + "CODEX_HOME": str(tmp_path / "codex"), + "CODEX_SQLITE_HOME": str(tmp_path / "native"), + "CODEX_STATE_DB": "", + } + timestamp = datetime.now(timezone.utc) + rollout = _rollout( + tmp_path, + "shared-parent", + [ + _event(timestamp, "turn_context", {"turn_id": "original-turn", "model": "gpt-5.6-sol"}), + ], + ) + _state_graph(environment, {"shared-parent": rollout}, []) + target = tmp_path / "target" + target.mkdir() + begun = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + owner = begun["usageOwner"] + assert owner["threadId"] == "shared-parent" + assert owner["turnId"] == "original-turn" + assert owner["dedicated"] is False + rollout.write_text( + rollout.read_text() + + json.dumps( + _event(timestamp, "turn_context", {"turn_id": "later-turn", "model": "gpt-6-astra"}) + ) + + "\n" + ) + joined = run_workbench( + state, + "begin-deep-scan", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + environment=environment, + )["deepScan"] + assert joined["usageOwner"] == owner + claim_args = [ + "claim-deep-scan-coordinator", + "--scan-id", + begun["scanId"], + "--thread-id", + "shared-parent", + ] + claimed = run_workbench(state, *claim_args, environment=environment)["deepScan"] + with sqlite3.connect(state / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET updated_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?", + (begun["scanId"],), + ) + recovered = run_workbench(state, *claim_args, environment=environment)["deepScan"] + assert recovered["coordinatorGeneration"] == claimed["coordinatorGeneration"] + 1 + assert recovered["usageOwner"] == owner + other_target = tmp_path / "other-target" + other_target.mkdir() + other = run_workbench( + state, + "begin-deep-scan", + "--thread-id", + "shared-parent", + "--target-path", + str(other_target), + "--scope", + ".", + "--scan-root", + str(tmp_path / "scans"), + environment=environment, + )["deepScan"] + assert other["usageOwner"]["turnId"] == "later-turn" + original = run_workbench( + state, "get-scan", "--scan-id", begun["scanId"], environment=environment + )["scan"] + assert original["executionAttribution"]["owner"] == owner diff --git a/plugins/codex-security/tests/test_finalization_selection_process_loss.py b/plugins/codex-security/tests/test_finalization_selection_process_loss.py new file mode 100644 index 000000000..c5ffafa61 --- /dev/null +++ b/plugins/codex-security/tests/test_finalization_selection_process_loss.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import io +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan +from test_publication_stop_interleavings import published_bytes, stop_scan + +_CRASH_SELECTION = """ +import io, json, os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="selection_commit_crash_test") +deep = api["deep_scan"] +deep.configure(deep.DeepScanDependencies(**{ + name: api["preserve_stopped_results_after_transition" + if name == "preserve_stopped_results" else name] + for name in deep.DeepScanDependencies.__dataclass_fields__ +})) +class CrashConnection(sqlite3.Connection): + def commit(self): + if sys.argv[4] == "before": + os._exit(72) + super().commit() + os._exit(73) +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +sys.stdin = io.StringIO(json.dumps({"resultPath": sys.argv[5]})) +deep.finish_deep_scan(connection, Namespace(**json.loads(sys.argv[3])), select_finalization=True) +raise AssertionError("selection never reached its commit boundary") +""" + + +@pytest.mark.parametrize("reason", ["saturated", "capped"]) +@pytest.mark.parametrize("cut", ["before", "after"]) +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize("stop_before_replay", [False, True]) +def test_selection_commit_loss_replays_accepted_identity_before_stopping( + workbench_api, + workbench_db, + publication_scan, + tmp_path, + monkeypatch, + reason, + cut, + cause, + stop_before_replay, +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + omissions = [] + if reason == "saturated": + omitted = add_worker(workbench_db, scan) + omitted.write_text(json.dumps(save_disposition(scan, omitted.parent, "reported"))) + save_disposition(scan, omitted.parent, "rejected") + omissions.append(omitted.parent.name) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "status = 'running', phase = 'reducing', terminal_reason = NULL, completed_at = NULL, " + "consecutive_no_new = stop_after_no_new, discovery_runs_dispatched = max_discovery_runs " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + args = Namespace( + scan_id=scan.scan_id, + coordinator_generation=3, + terminal_reason=reason, + manifest_path=str(scan.scan_dir / "scan-manifest.json"), + staged_manifest_path=None, + omitted_worker_id=omissions, + ) + database = tmp_path / "selection.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + before = published_bytes(scan) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_SELECTION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database), + json.dumps(vars(args)), + cut, + str(result), + ], + capture_output=True, + text=True, + ) + assert child.returncode == (72 if cut == "before" else 73), (child.stdout, child.stderr) + assert published_bytes(scan) == before + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + selected = json.loads(run["finalization_input_json"]) if cut == "after" else None + if cut == "before": + assert run["finalization_input_json"] is None + assert run["terminal_reason"] is None + # The request names the deleted output. Selection resolves its committed + # accepted attempt rather than reading that replaceable file again. + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps({"resultPath": str(result)}))) + if stop_before_replay: + stop_scan(workbench_api, connection, scan, cause) + stopped = published_bytes(scan) + stopped_database = "\n".join(connection.iterdump()) + with pytest.raises(SystemExit, match="running|stopped|failed|canceled"): + workbench_api["deep_scan"].finish_deep_scan( + connection, args, select_finalization=True + ) + assert "\n".join(connection.iterdump()) == stopped_database + assert published_bytes(scan) == stopped + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + assert run["terminal_reason"] == (reason if cut == "after" else None) + assert ( + json.loads(run["finalization_input_json"]) if cut == "after" else None + ) == selected + assert accepted.read_bytes() == before[accepted.relative_to(scan.scan_dir).as_posix()] + return + replayed = workbench_api["deep_scan"].finish_deep_scan( + connection, args, select_finalization=True + )["deepScan"] + selection = replayed["finalizationInput"] + if selected is not None: + assert selection == selected + assert selection["resultPath"] == accepted.relative_to(scan.scan_dir).as_posix() + assert selection["resultSha256"] == accepted.stem + assert selection["terminalReason"] == reason + assert selection["omittedWorkerIds"] == omissions + assert replayed["terminalReason"] == reason + assert replayed["status"] == "running" + assert published_bytes(scan) == before + stale = stage_publication(scan, generation=2, result_path=accepted, title="Stale aggregate") + with pytest.raises(SystemExit, match="generation"): + workbench_api["write_scan_draft"](connection, stale) + assert published_bytes(scan) == before + stop_scan(workbench_api, connection, scan, cause) + stopped = published_bytes(scan) + late = stage_publication(scan, generation=3, result_path=accepted, title="Late aggregate") + with pytest.raises(SystemExit, match="stopped"): + workbench_api["write_scan_draft"](connection, late) + assert published_bytes(scan) == stopped + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + assert run["terminal_reason"] == reason + assert json.loads(run["finalization_input_json"]) == selection + assert accepted.read_bytes() == before[selection["resultPath"]] + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert findings[0].get("extensions", {}).get("candidateId") != "candidate-disposition" diff --git a/plugins/codex-security/tests/test_publication_stop_interleavings.py b/plugins/codex-security/tests/test_publication_stop_interleavings.py new file mode 100644 index 000000000..d1f4c8323 --- /dev/null +++ b/plugins/codex-security/tests/test_publication_stop_interleavings.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +import sys +from argparse import Namespace +from pathlib import Path + +import pytest +from test_accepted_publication_references import accept_reducer +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def saved_selection(connection, scan, accepted, omitted=None, *, reason="saturated"): + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": accepted.stem, + "terminalReason": reason, + "omittedWorkerIds": [omitted.parent.name] if omitted is not None else [], + "selectedAt": scan.timestamp, + } + with connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?, terminal_reason = ?, phase = 'terminal' " + "WHERE scan_id = ?", + (json.dumps(selection), reason, scan.scan_id), + ) + return selection + + +def stop_scan(api, connection, scan, cause): + if cause == "cancel": + return api["cancel_scan"](connection, Namespace(scan_id=scan.scan_id, thread_id=None)) + return api["fail_scan"]( + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Scan stopped after reaching the configured cost limit.", + ), + ) + + +def published_bytes(scan): + return { + path.relative_to(scan.scan_dir).as_posix(): path.read_bytes() + for path in scan.scan_dir.rglob("*") + if path.is_file() and "drafts" not in path.relative_to(scan.scan_dir).parts + } + + +_CRASH_STOPPED_PUBLICATION = """ +import os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="stopped_publication_crash_test") +scan_id, cause, boundary = sys.argv[3:] + +class CrashConnection(sqlite3.Connection): + def __exit__(self, *args): + sealing = self.execute( + "SELECT seal_manifest_digest FROM scans WHERE id = ?", (scan_id,) + ).fetchone()[0] is not None + if sealing and boundary == "sqlite-before": + os._exit(72) + result = super().__exit__(*args) + if sealing and boundary == "sqlite-after": + os._exit(73) + return result + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +import finalize_scan_contract as contract +original_write = contract.write_scan_local_bytes +def crash_after_write(root, relative, contents, **kwargs): + original_write(root, relative, contents, **kwargs) + if relative == boundary: + os._exit(71) +contract.write_scan_local_bytes = crash_after_write +if cause == "cancel": + api["cancel_scan"](connection, Namespace(scan_id=scan_id, thread_id=None)) +else: + api["fail_scan"](connection, Namespace( + scan_id=scan_id, claim_token=None, cost_json=None, + message="Scan stopped after reaching the configured cost limit." + )) +raise AssertionError("stop never reached the requested publication boundary") +""" + +_CRASH_SELECTION_RECOVERY = """ +import os, runpy, sqlite3, sys +from argparse import Namespace + +api = runpy.run_path(sys.argv[1], run_name="selection_recovery_crash_test") +deep = api["deep_scan"] +deep.configure(deep.DeepScanDependencies(**{ + name: api["preserve_stopped_results_after_transition" + if name == "preserve_stopped_results" else name] + for name in deep.DeepScanDependencies.__dataclass_fields__ +})) + +class CrashConnection(sqlite3.Connection): + def commit(self): + if sys.argv[4] == "before": + os._exit(72) + super().commit() + os._exit(73) + +connection = sqlite3.connect(sys.argv[2], factory=CrashConnection) +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +deep.claim_deep_scan_coordinator(connection, Namespace( + scan_id=sys.argv[3], thread_id="fixture-owner", + claim_token=None, coordinator_generation=None, +)) +raise AssertionError("recovery never reached the requested commit boundary") +""" + + +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize("cut", ["before-selection", "selected", "published", "sealed"]) +def test_stop_and_publication_keep_the_winning_terminal_outcome( + workbench_api, workbench_db, publication_scan, tmp_path, cause, cut +): + scan = publication_scan() + _, accepted, coverage = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + rejected = save_disposition(scan, omitted.parent, "reported") + omitted.write_text(json.dumps(rejected)) + save_disposition(scan, omitted.parent, "rejected") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + if cut in {"before-selection", "selected"}: + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + selection = ( + None + if cut == "before-selection" + else saved_selection(workbench_db, scan, accepted, omitted) + ) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Selected accepted aggregate" + ) + # The stop path must recover the accepted bytes and the rejection disposition. + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + evidence = {accepted: accepted.read_bytes(), omitted: omitted.read_bytes()} + database_path = tmp_path / "stop-publication.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut in {"published", "sealed"}: + workbench_api["write_scan_draft"](connection, staged) + complete_args = Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + before_stop = published_bytes(scan) + if cut == "sealed": + with pytest.raises(SystemExit, match="running|completed"): + stop_scan(workbench_api, connection, scan, cause) + assert published_bytes(scan) == before_stop + else: + stop_scan(workbench_api, connection, scan, cause) + + # Reconnect after the winning commit, then deliver the old publisher response. + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = dict(connection.execute("SELECT * FROM scans").fetchone()) + run = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + assert row["status"] == ("complete" if cut == "sealed" else "failed") + assert bool(row["canceled_at"]) == (cause == "cancel" and cut != "sealed") + if cause == "cost" and cut != "sealed": + assert row["failure_message"] == ( + "Scan stopped after reaching the configured cost limit." + ) + assert json.loads(run["finalization_input_json"] or "null") == selection + if selection is not None: + assert run["terminal_reason"] == selection["terminalReason"] + if cut != "sealed" and cause == "cancel": + assert run["status"] == "canceled" + if cut in {"published", "sealed"}: + assert run["terminal_reason"] == "saturated" + frozen = published_bytes(scan) + with pytest.raises(SystemExit, match="stopped"): + workbench_api["write_scan_draft"](connection, staged) + if cut == "sealed": + workbench_api["complete_scan"](connection, complete_args) + else: + with pytest.raises(SystemExit): + workbench_api["complete_scan"](connection, complete_args) + workbench_api["preserve_scan_results"]( + connection, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + assert published_bytes(scan) == frozen + assert dict(connection.execute("SELECT * FROM scans").fetchone()) == row + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert all(path.read_bytes() == contents for path, contents in evidence.items()) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert all( + finding.get("extensions", {}).get("candidateId") != "candidate-disposition" + for finding in findings + ) + if cut != "sealed": + published_coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["deferred"][0] in published_coverage["deferred"] + assert published_coverage["completeness"] == "partial" + + +@pytest.mark.parametrize("cause", ["cancel", "cost"]) +@pytest.mark.parametrize( + "boundary", + ["findings.json", "coverage.json", "scan-manifest.json", "sqlite-before", "sqlite-after"], +) +def test_stopped_publication_process_loss_keeps_frozen_rejection_and_original_selection( + workbench_api, workbench_db, publication_scan, tmp_path, cause, boundary +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + reported = save_disposition(scan, omitted.parent, "reported") + omitted.write_text(json.dumps(reported)) + save_disposition(scan, omitted.parent, "rejected") + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', phase = 'reducing', " + "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?", + (scan.scan_id,), + ) + selection = saved_selection(workbench_db, scan, accepted, omitted) + staged = stage_publication( + scan, generation=3, result_path=accepted, title="Obsolete selected publication" + ) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + evidence = {accepted: accepted.read_bytes(), omitted: omitted.read_bytes()} + database_path = tmp_path / "stopped-crash.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_STOPPED_PUBLICATION, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + scan.scan_id, + cause, + boundary, + ], + capture_output=True, + text=True, + ) + assert child.returncode == {"sqlite-before": 72, "sqlite-after": 73}.get(boundary, 71), ( + child.stdout, + child.stderr, + ) + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + row = connection.execute("SELECT * FROM scans").fetchone() + assert row["status"] == "failed" + assert bool(row["canceled_at"]) == (cause == "cancel") + assert bool(row["seal_manifest_digest"]) == (boundary == "sqlite-after") + frozen_sources = row["retained_source_digests_json"] + frozen_heads = row["retained_checkpoint_heads_json"] + assert frozen_sources and frozen_heads + run = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + workers = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")] + assert json.loads(run["finalization_input_json"]) == selection + assert run["terminal_reason"] == selection["terminalReason"] + assert run["status"] == ("canceled" if cause == "cancel" else "failed") + # The replacement process sees a different live head, but replays the + # already committed stopped selection instead of restoring the candidate. + save_disposition(scan, omitted.parent, "reported") + interrupted = published_bytes(scan) + with pytest.raises(SystemExit, match="stopped"): + workbench_api["write_scan_draft"](connection, staged) + assert published_bytes(scan) == interrupted + args = Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ) + workbench_api["preserve_scan_results"](connection, args) + row = connection.execute("SELECT * FROM scans").fetchone() + assert row["status"] == "failed" + assert bool(row["canceled_at"]) == (cause == "cancel") + assert row["retained_source_digests_json"] == frozen_sources + assert row["retained_checkpoint_heads_json"] == frozen_heads + assert row["seal_manifest_digest"] + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == run + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert len(findings) == 1 + assert findings[0].get("extensions", {}).get("candidateId") != "candidate-disposition" + assert all(path.read_bytes() == contents for path, contents in evidence.items()) + sealed = published_bytes(scan) + workbench_api["preserve_scan_results"](connection, args) + assert published_bytes(scan) == sealed + assert connection.execute("SELECT COUNT(*) FROM finding_occurrences").fetchone()[0] == 1 + + +@pytest.mark.parametrize("cut", ["before", "after"]) +def test_interrupted_selection_recovery_fences_observers_and_keeps_original_deadline( + workbench_api, workbench_db, publication_scan, tmp_path, cut +): + scan = publication_scan() + _, accepted, _ = accept_reducer(workbench_db, scan) + omitted = add_worker(workbench_db, scan) + selection = saved_selection(workbench_db, scan, accepted, omitted) + with workbench_db: + workbench_db.execute( + "UPDATE scans SET deep_scan_owner_thread_id = 'fixture-owner' WHERE id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET status = 'running', " + "completed_at = NULL, max_time_hours = 1, " + "created_at = '2000-01-01T00:00:00Z', updated_at = '2000-01-01T00:00:00Z' " + "WHERE scan_id = ?", + (scan.scan_id,), + ) + workbench_db.execute( + "UPDATE deep_scan_workers SET merge_state = 'buffered' WHERE id = ?", + (omitted.parent.name,), + ) + database_path = tmp_path / "recovery.sqlite3" + with sqlite3.connect(database_path) as connection: + workbench_db.backup(connection) + original = "\n".join(connection.iterdump()) + before = published_bytes(scan) + child = subprocess.run( + [ + sys.executable, + "-c", + _CRASH_SELECTION_RECOVERY, + str(Path(__file__).resolve().parents[1] / "scripts" / "workbench_db.py"), + str(database_path), + scan.scan_id, + cut, + ], + capture_output=True, + text=True, + ) + assert child.returncode == (72 if cut == "before" else 73), (child.stdout, child.stderr) + deep = workbench_api["deep_scan"] + args = Namespace( + scan_id=scan.scan_id, + thread_id="fixture-owner", + claim_token=None, + coordinator_generation=None, + ) + with sqlite3.connect(database_path) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + if cut == "before": + assert "\n".join(connection.iterdump()) == original + run = connection.execute("SELECT * FROM deep_scan_runs").fetchone() + assert run["coordinator_generation"] == (3 if cut == "before" else 4) + workers = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers")] + attempts = [dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts")] + replayed = deep.claim_deep_scan_coordinator(connection, args) + assert replayed["coordinatorDisposition"] == ("adopted" if cut == "before" else "observing") + assert replayed["deepScan"]["coordinatorGeneration"] == 4 + assert replayed["deepScan"]["finalizationInput"] == selection + assert replayed["deepScan"]["createdAt"] == "2000-01-01T00:00:00Z" + assert replayed["deepScan"]["config"]["maxTimeHours"] == 1 + assert replayed["deepScan"]["phase"] == "terminal" + assert replayed["deepScan"]["terminalReason"] == selection["terminalReason"] + assert deep.deep_scan_deadline_reached( + connection.execute("SELECT * FROM deep_scan_runs").fetchone() + ) + stable = "\n".join(connection.iterdump()) + stale_claim = Namespace(**{**vars(args), "coordinator_generation": 3}) + with pytest.raises(SystemExit, match="generation"): + deep.claim_deep_scan_coordinator(connection, stale_claim) + assert "\n".join(connection.iterdump()) == stable + stale = stage_publication(scan, generation=3, result_path=accepted, title="Old coordinator") + with pytest.raises(SystemExit, match="generation"): + workbench_api["write_scan_draft"](connection, stale) + assert published_bytes(scan) == before + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_workers") + ] == workers + assert [ + dict(row) for row in connection.execute("SELECT * FROM deep_scan_attempts") + ] == attempts + current = stage_publication( + scan, generation=4, result_path=accepted, title="Recovered selected aggregate" + ) + workbench_api["write_scan_draft"](connection, current) + assert accepted.read_bytes() == before[accepted.relative_to(scan.scan_dir).as_posix()] + assert "\n".join(connection.iterdump()) == stable diff --git a/plugins/codex-security/tests/test_report_projection.py b/plugins/codex-security/tests/test_report_projection.py index e88dcb63a..f5881a3a5 100644 --- a/plugins/codex-security/tests/test_report_projection.py +++ b/plugins/codex-security/tests/test_report_projection.py @@ -75,6 +75,26 @@ def test_projection_normalizes_multiline_and_block_structural_text() -> None: assert "Text: ## Injected remediation - unsafe instruction" in markdown +def test_linked_writeup_retains_distinct_source_fixes() -> None: + manifest, findings, coverage = canonical_documents() + finding = findings["findings"][0] + finding["writeup"] = {"reportPath": "findings/parser/parser.md"} + finding["remediation"] = "Validate the record length." + finding["provenance"] = { + "sourceFindings": [ + {"id": "review-1:0", "finding": {"remediation": "Validate the record length."}}, + {"id": "review-2:0", "finding": {"remediation": "Reject duplicate record keys."}}, + {"id": "review-3:0", "finding": {"remediation": "Reject duplicate record keys."}}, + ] + } + + markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) + + assert "findings/parser/parser.md" in markdown + assert markdown.count("Validate the record length.") == 1 + assert markdown.count("Reject duplicate record keys.") == 1 + + def test_projection_renders_inline_code_and_section_code_evidence() -> None: manifest, findings, coverage = canonical_documents() finding = findings["findings"][0] @@ -744,8 +764,8 @@ def test_projection_explains_unvalidated_findings_after_cost_limit(reason: str) assert "| Reportable findings | 0 |" in markdown assert "| Coverage | partial |" in markdown assert ( - "No findings were validated before the scan reached its cost limit. " - "Review the deferred candidates in Open Questions And Follow Up." + "No findings are included in this partial report. " + "Review the unresolved work in Open Questions And Follow Up." ) in markdown assert "No reportable findings survived" not in markdown assert "## Open Questions And Follow Up" in markdown @@ -870,7 +890,22 @@ def test_projection_keeps_deferred_follow_up_with_open_questions() -> None: assert "Surfaces: parser-surface." in markdown -def test_projection_includes_surface_evidence_receipts() -> None: +@pytest.mark.parametrize( + ("details", "expected"), + [ + ({"notes": "Reviewed parser entrypoints."}, "Reviewed parser entrypoints."), + ({"reason": "Caller policy is unknown."}, "Caller policy is unknown."), + ( + {"notes": "Reviewed parser entrypoints.", "reason": "Caller policy is unknown."}, + "Reviewed parser entrypoints. Caller policy is unknown.", + ), + ({"notes": "Policy checked.", "reason": "Policy checked."}, "Policy checked."), + ({"notes": ""}, ""), + ({"notes": " \t "}, ""), + ({}, "No additional canonical notes were recorded."), + ], +) +def test_projection_includes_surface_evidence_receipts(details: dict, expected: str) -> None: manifest, findings, coverage = canonical_documents() coverage["surfaces"] = [ { @@ -878,10 +913,13 @@ def test_projection_includes_surface_evidence_receipts() -> None: "label": "Parser", "disposition": "no_issue_found", "receiptRefs": ["artifacts/receipts/parser.jsonl"], - "notes": "Reviewed parser entrypoints.", + **details, } ] markdown = PROJECTION.build_report_markdown(manifest, findings, coverage) - assert "Reviewed parser entrypoints. Evidence: artifacts/receipts/parser.jsonl" in markdown + row = next(line for line in markdown.splitlines() if "artifacts/receipts/parser.jsonl" in line) + assert row.endswith( + f"| {expected + ' ' if expected else ''}Evidence: artifacts/receipts/parser.jsonl |" + ) diff --git a/plugins/codex-security/tests/test_selected_publication_authority.py b/plugins/codex-security/tests/test_selected_publication_authority.py new file mode 100644 index 000000000..b3bd24ea2 --- /dev/null +++ b/plugins/codex-security/tests/test_selected_publication_authority.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import hashlib +import json + +import pytest +from test_deep_scan_publication_authority import stage_publication +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("publication", ["selected", "mutable", "stale-generation", "unfenced"]) +def test_publication_uses_committed_finalization_selection( + workbench_api, workbench_db, publication_scan, publication +): + scan = publication_scan() + result = add_worker(workbench_db, scan) + contents = json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": scan.findings, + "coverage": scan.coverage, + } + ).encode() + digest = hashlib.sha256(contents).hexdigest() + accepted = result.parent / "accepted" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(contents) + selection = { + "version": 1, + "resultPath": accepted.relative_to(scan.scan_dir).as_posix(), + "resultSha256": digest, + "terminalReason": "saturated", + "omittedWorkerIds": [], + "selectedAt": scan.timestamp, + } + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(result),), + ) + workbench_db.execute( + "UPDATE deep_scan_runs SET coordinator_generation = ?, finalization_input_json = ?, " + "workflow_version = 'deep-security-scan/v2' " + "WHERE scan_id = ?", + (1 if publication == "unfenced" else 3, json.dumps(selection), scan.scan_id), + ) + # The accepted bytes survive replacement or deletion of the worker's output. + result.unlink(missing_ok=True) + staged = stage_publication( + scan, + generation=None + if publication == "unfenced" + else 2 + if publication == "stale-generation" + else 3, + result_path=result if publication == "mutable" else accepted, + title="Selected accepted aggregate", + ) + before = {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} + + if publication == "selected": + workbench_api["write_scan_draft"](workbench_db, staged) + findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"] + assert findings[0]["title"] == "Selected accepted aggregate" + else: + with pytest.raises(SystemExit, match="coordinator|publication|aggregate"): + workbench_api["write_scan_draft"](workbench_db, staged) + assert {path: path.read_bytes() for path in scan.scan_dir.rglob("*.json")} == before + assert accepted.read_bytes() == contents + assert ( + json.loads( + workbench_db.execute( + "SELECT finalization_input_json FROM deep_scan_runs WHERE scan_id = ?", + (scan.scan_id,), + ).fetchone()[0] + ) + == selection + ) diff --git a/plugins/codex-security/tests/test_stopped_accepted_digests.py b/plugins/codex-security/tests/test_stopped_accepted_digests.py new file mode 100644 index 000000000..3777f85c0 --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_accepted_digests.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sqlite3 +from argparse import Namespace + +import pytest +from test_accepted_publication_references import accept_reducer +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("recorded", [True, False], ids=["accepted-receipt", "legacy"]) +@pytest.mark.parametrize("changed", [False, True], ids=["original", "changed"]) +@pytest.mark.parametrize("historical", [False, True], ids=["current-attempt", "prior-attempt"]) +def test_stopped_recovery_checks_recorded_accepted_bytes( + workbench_api, workbench_db, publication_scan, tmp_path, recorded, changed, historical +): + scan = publication_scan() + result, accepted, _ = accept_reducer(workbench_db, scan) + checkpoint = result.parent / "checkpoints" / accepted.name + checkpoint.parent.mkdir() + accepted.rename(checkpoint) + original = checkpoint.read_bytes() + digest = hashlib.sha256(original).hexdigest() + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_attempts SET accepted_result_path = ? WHERE worker_id = ?", + (str(checkpoint), result.parent.name), + ) + if not recorded: + workbench_db.execute( + "DELETE FROM deep_scan_attempts WHERE worker_id = ?", (result.parent.name,) + ) + if historical: + workbench_db.execute( + "UPDATE deep_scan_workers SET attempt = 2, status = 'running' WHERE id = ?", + (result.parent.name,), + ) + if changed: + damaged = json.loads(original) + damaged["findings"][0]["summary"] = "Unaccepted changed evidence." + checkpoint.write_text(json.dumps(damaged)) + accepted_bytes = checkpoint.read_bytes() + # An unrelated valid source must still survive stopped partial preservation. + healthy = add_worker(workbench_db, scan) + finding = copy.deepcopy(scan.findings[0]) + finding["summary"] = "Independent preserved evidence." + finding["identity"]["anchor"] += ".independent" + finding["locations"][0]["startLine"] = 2 + finding["locations"][0]["endLine"] = 2 + healthy.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [finding], + "coverage": scan.coverage, + } + ) + ) + healthy_bytes = healthy.read_bytes() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + database = tmp_path / "accepted-digests.sqlite3" + with sqlite3.connect(database) as connection: + workbench_db.backup(connection) + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + stopped = workbench_api["fail_scan"]( + connection, + Namespace( + scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped." + ), + )["scan"] + summaries = {finding["summary"] for finding in stopped["findings"]} + assert "Independent preserved evidence." in summaries + assert ("Unaccepted changed evidence." in summaries) == (changed and not recorded) + if not changed: + assert scan.findings[0]["summary"] in summaries + if recorded: + assert ( + connection.execute( + "SELECT accepted_result_sha256 FROM deep_scan_attempts WHERE worker_id = ?", + (result.parent.name,), + ).fetchone()[0] + == digest + ) + if recorded and changed: + warnings = json.loads( + connection.execute("SELECT completion_warnings_json FROM scans").fetchone()[0] + ) + assert any("changed after acceptance" in warning for warning in warnings) + assert checkpoint.read_bytes() == accepted_bytes + assert healthy.read_bytes() == healthy_bytes diff --git a/plugins/codex-security/tests/test_stopped_result_version_boundary.py b/plugins/codex-security/tests/test_stopped_result_version_boundary.py new file mode 100644 index 000000000..f31e5c0ee --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_result_version_boundary.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import hashlib +import json +from argparse import Namespace + +import pytest +from test_checkpoint_publication_authority import save_disposition +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +def snapshot(connection, scan_dir): + return { + "database": "\n".join(connection.iterdump()), + "files": { + path.relative_to(scan_dir).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in scan_dir.rglob("*") + if path.is_file() + }, + } + + +@pytest.mark.parametrize("operation", ["preserve", "recover"]) +@pytest.mark.parametrize( + "protocol", + ["supported", "supported-v2", "supported-mcp-v1", "future-workflow", "future-selection"], +) +def test_stopped_result_publication_requires_supported_protocol( + workbench_api, workbench_db, publication_scan, monkeypatch, operation, protocol +): + scan = publication_scan() + supported_workflows = { + "supported": "deep-security-scan/v1", + "supported-v2": "deep-security-scan/v2", + "supported-mcp-v1": "deep-scan-mcp/v1", + } + if protocol in supported_workflows: + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = ? WHERE scan_id = ?", + (supported_workflows[protocol], scan.scan_id), + ) + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + result = add_worker(workbench_db, scan, status="canceled") + draft = save_disposition(scan, result.parent, "reported") + result.write_text(json.dumps(draft)) + + def interrupt_publication(*args, **kwargs): + raise OSError("Synthetic publication interruption") + + with monkeypatch.context() as patch: + patch.setattr( + workbench_api["saved_results"], + "_write_prepared_scan_finalization", + interrupt_publication, + ) + workbench_api["fail_scan"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + cost_json=None, + message="Original worker stop.", + ), + ) + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["status"] == "failed" + assert row["retained_source_digests_json"] + assert row["seal_manifest_digest"] is None + + with workbench_db: + if protocol == "future-workflow": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'future/v99' WHERE scan_id = ?", + (scan.scan_id,), + ) + elif protocol == "future-selection": + workbench_db.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ? WHERE scan_id = ?", + (json.dumps({"version": 99}), scan.scan_id), + ) + before = snapshot(workbench_db, scan.scan_dir) + error = None + try: + if operation == "preserve": + workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, + claim_token=None, + thread_id=None, + coordinator_generation=None, + ), + ) + else: + workbench_api["recover_scan_results"](workbench_db, Namespace(scan_id=scan.scan_id)) + except SystemExit as failure: + error = str(failure) + after = snapshot(workbench_db, scan.scan_dir) + changed_files = sorted( + path + for path in before["files"].keys() | after["files"].keys() + if before["files"].get(path) != after["files"].get(path) + ) + print( + json.dumps( + { + "operation": operation, + "protocol": protocol, + "error": error, + "database_changed": before["database"] != after["database"], + "changed_files": changed_files, + } + ) + ) + if protocol in supported_workflows: + assert error is None + assert before != after + row = workbench_db.execute("SELECT * FROM scans WHERE id = ?", (scan.scan_id,)).fetchone() + assert row["seal_manifest_digest"] + assert row["failure_message"] == "Original worker stop." + run = workbench_db.execute( + "SELECT workflow_version FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,) + ).fetchone() + assert run["workflow_version"] == supported_workflows[protocol] + else: + assert error is not None and "unsupported" in error.lower() + assert after == before diff --git a/plugins/codex-security/tests/test_stopped_source_coverage.py b/plugins/codex-security/tests/test_stopped_source_coverage.py new file mode 100644 index 000000000..57b8e6534 --- /dev/null +++ b/plugins/codex-security/tests/test_stopped_source_coverage.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import copy +import json +from argparse import Namespace + +import pytest +from test_deep_scan_successful_publication import add_worker +from test_deep_scan_successful_publication import publication_scan as publication_scan + + +@pytest.mark.parametrize("host_coverage", [True, False], ids=["accepted-projection", "legacy"]) +@pytest.mark.parametrize("parent_draft", [True, False], ids=["parent-draft", "no-parent"]) +def test_stopped_recovery_preserves_accepted_coverage_without_worker_id_collisions( + workbench_api, workbench_db, publication_scan, host_coverage, parent_draft +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + source_coverage = { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + "reviews": [], + } + source_files = [] + for disposition in ("needs_follow_up", "rejected"): + result = add_worker(workbench_db, scan) + worker_id = result.parent.name + surface = { + "id": "surface-1", + "candidateId": "candidate-1", + "label": "Independent review", + "disposition": disposition, + "receiptRefs": [], + } + deferred = {"candidateId": "candidate-1", "reason": "Validation remains unresolved."} + coverage = { + "completeness": "partial" if disposition == "needs_follow_up" else "complete", + "surfaces": [surface], + "explicitExclusions": [], + "deferred": [deferred] if disposition == "needs_follow_up" else [], + } + result.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": coverage} + ) + ) + source_files.append(result) + prefix = f"{worker_id}-attempt-1" + provenance = {"workerId": worker_id, "attempt": 1, "candidateId": "candidate-1"} + source_coverage["reviews"].append( + {"workerId": worker_id, "attempt": 1, "completeness": coverage["completeness"]} + ) + source_coverage["surfaces"].append( + { + **surface, + "id": f"{prefix}-surface-1", + "provenance": {**provenance, "sourceId": "surface-1"}, + } + ) + if coverage["deferred"]: + source_coverage["deferred"].append( + { + **deferred, + "id": f"{prefix}-deferred-1", + "candidateId": f"{prefix}-candidate-1", + "provenance": provenance, + } + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + aggregate = {"scanId": scan.scan_id, "complete": True, "findings": []} + if host_coverage: + aggregate["sourceCoverage"] = copy.deepcopy(source_coverage) + reducer.write_text(json.dumps(aggregate)) + source_files.append(reducer) + saved_bytes = {path: path.read_bytes() for path in source_files} + if not parent_draft: + for filename in ("scan-manifest.json", "findings.json", "coverage.json"): + (scan.scan_dir / filename).unlink() + + stopped = workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + )["scan"] + + assert stopped["progress"]["status"] == "failed" + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert len(coverage["deferred"]) == 2 + assert coverage["deferred"][-1]["id"] == "scan-stopped" + assert len(coverage["surfaces"]) == 2 + if host_coverage: + for field in ("reviews", "surfaces", "deferred"): + assert ( + coverage[field][:-1] if field == "deferred" else coverage[field] + ) == source_coverage[field] + else: + assert coverage["deferred"][0]["candidateId"] == "candidate-1" + manifest = (scan.scan_dir / "scan-manifest.json").read_bytes() + workbench_api["preserve_scan_results"]( + workbench_db, + Namespace( + scan_id=scan.scan_id, claim_token=None, thread_id=None, coordinator_generation=None + ), + ) + assert (scan.scan_dir / "scan-manifest.json").read_bytes() == manifest + assert all(path.read_bytes() == contents for path, contents in saved_bytes.items()) + + +def test_stopped_recovery_keeps_unmerged_coverage_after_accepted_review( + workbench_api, workbench_db, publication_scan +): + scan = publication_scan() + (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []})) + accepted = add_worker(workbench_db, scan) + accepted.write_text( + json.dumps( + {"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": scan.coverage} + ) + ) + reducer = add_worker(workbench_db, scan) + with workbench_db: + workbench_db.execute( + "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' " + "WHERE result_manifest_path = ?", + (str(reducer),), + ) + reviews = [{"workerId": accepted.parent.name, "attempt": 1, "completeness": "complete"}] + reducer.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": True, + "findings": [], + "sourceCoverage": {**scan.coverage, "reviews": reviews}, + } + ) + ) + pending = add_worker(workbench_db, scan, status="canceled") + deferred = {"id": "pending-review", "reason": "The independent review remains unresolved."} + pending.write_text( + json.dumps( + { + "scanId": scan.scan_id, + "complete": False, + "findings": [], + "coverage": {**scan.coverage, "completeness": "partial", "deferred": [deferred]}, + } + ) + ) + + workbench_api["fail_scan"]( + workbench_db, + Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Audit stopped."), + ) + + coverage = json.loads((scan.scan_dir / "coverage.json").read_text()) + assert coverage["completeness"] == "partial" + assert coverage["reviews"] == reviews + assert deferred in coverage["deferred"] diff --git a/plugins/codex-security/tests/test_workbench_completion_binding.py b/plugins/codex-security/tests/test_workbench_completion_binding.py index dbcdafd9f..357cf0044 100644 --- a/plugins/codex-security/tests/test_workbench_completion_binding.py +++ b/plugins/codex-security/tests/test_workbench_completion_binding.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +import pytest from workbench_test_support import ( create_saved_workspace, initialize_git_repository, @@ -69,6 +70,40 @@ def _start_deep_scan_with_draft_findings(tmp_path: Path) -> tuple[Path, str, Pat return state_dir, scan_id, scan_dir +@pytest.mark.parametrize("completeness", ["partial", "complete", "unknown"]) +def test_deep_completion_returns_sealed_coverage_on_replay( + tmp_path: Path, completeness: str +) -> None: + state_dir, scan_id, scan_dir = _start_deep_scan_with_draft_findings(tmp_path) + coverage_path = scan_dir / "coverage.json" + coverage = json.loads(coverage_path.read_text()) + coverage["completeness"] = completeness + coverage["deferred"] = ( + [{"id": "external-control", "reason": "Deployment configuration is unavailable."}] + if completeness == "partial" + else [] + ) + coverage_path.write_text(json.dumps(coverage)) + completed = run_workbench(state_dir, "complete-scan", "--scan-id", scan_id) + canonical = json.loads(coverage_path.read_text()) + assert canonical["completeness"] == completeness + assert len(canonical["deferred"]) == (1 if completeness == "partial" else 0) + expected = { + "completeness": canonical["completeness"], + "surfaceCount": len(canonical["surfaces"]), + "deferredCount": len(canonical["deferred"]), + "explicitExclusionCount": len(canonical["explicitExclusions"]), + } + artifacts = ["scan-manifest.json", "findings.json", "coverage.json", "report.md"] + sealed = [(scan_dir / name).read_bytes() for name in artifacts] + replayed = run_workbench(state_dir, "complete-scan", "--scan-id", scan_id) + for result in (completed, replayed): + assert result["scan"]["progress"]["status"] == "complete" + assert result["coverageSummary"] == expected + assert result["scan"]["usage"] == completed["scan"]["usage"] + assert [(scan_dir / name).read_bytes() for name in artifacts] == sealed + + def register_cli_scan(state_dir: Path, target: Path, scan_dir: Path) -> dict[str, Any]: scan_dir.mkdir(mode=0o700) return run_workbench( diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py index f68ccdef9..02f11d039 100644 --- a/plugins/codex-security/tests/test_workbench_db.py +++ b/plugins/codex-security/tests/test_workbench_db.py @@ -6,6 +6,7 @@ import runpy import sqlite3 import subprocess +import sys import time import uuid from concurrent.futures import ThreadPoolExecutor @@ -13,6 +14,7 @@ from typing import Any import pytest +import workbench_test_support from workbench_test_support import ( SCRIPT, create_saved_git_workspace, @@ -55,6 +57,9 @@ BUDGET_WARNING = "Scan stopped: estimated cost $0.00625 exceeded the $0.005 cost limit." EXPECTED_TABLES = { + "deep_scan_attempts", + "deep_scan_attempt_sessions", + "deep_scan_merge_claims", "deep_scan_dedup_inputs", "deep_scan_runs", "deep_scan_workers", @@ -169,6 +174,198 @@ def budget_scan_fixture( return state_dir, target, scan_dir, scan_id, ledger +@pytest.mark.parametrize("operation", ["complete-scan", "complete-budget-exhausted-scan"]) +@pytest.mark.parametrize("protocol", ["supported", "future-workflow", "future-selection"]) +def test_completion_rejects_unknown_protocol_before_mutation( + workbench_api, monkeypatch, tmp_path, operation, protocol +): + script = str(workbench_api["__file__"]) + monkeypatch.setattr(workbench_test_support, "SCRIPT", script) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + environment = {**os.environ, "CODEX_SECURITY_STATE_DIR": str(state_dir)} + cost_args = ["--scan-id", scan_id, "--cost-json", json.dumps(BUDGET_COST)] + if operation == "complete-scan": + cut_program = """ +import os, runpy, sys +script, *args = sys.argv[1:] +api = runpy.run_path(script, run_name="completion_version_test") +namespace = api["main"].__globals__ +original = namespace["_write_prepared_scan_finalization"] +def after_seal(*args, **kwargs): + original(*args, **kwargs) + os._exit(86) +namespace["_write_prepared_scan_finalization"] = after_seal +sys.argv = [script, *args] +api["main"]() +""" + cut = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + cut_program, + script, + "complete-budget-exhausted-scan", + *cost_args, + "--message", + BUDGET_WARNING, + ], + env=environment, + capture_output=True, + text=True, + ) + assert cut.returncode == 86, cut.stderr + assert json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["sealedAt"] + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("running",) + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + script, + operation, + *cost_args, + *([] if operation == "complete-scan" else ["--message", BUDGET_WARNING]), + ], + env=environment, + capture_output=True, + text=True, + ) + after = snapshot() + if protocol == "supported": + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["scan"]["progress"]["status"] == "complete" + if operation == "complete-scan": + assert after[1] == before[1] + else: + assert result.returncode != 0 + assert "unsupported" in result.stderr.lower() + assert after == before + + +@pytest.mark.parametrize("legacy_digest", [False, True]) +@pytest.mark.parametrize( + "protocol", ["supported", "legacy-no-run", "future-workflow", "future-selection"] +) +def test_completed_replay_checks_protocol_before_cost_or_digest_writes( + workbench_api, monkeypatch, tmp_path, protocol, legacy_digest +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path) + complete_budget_scan(state_dir, scan_id) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT status FROM scans").fetchone() == ("complete",) + if legacy_digest: + connection.execute("UPDATE scans SET seal_manifest_digest = NULL") + if protocol == "future-workflow": + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'future/v99'") + elif protocol == "future-selection": + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "finalization_input_json = ?", + (json.dumps({"version": 99}),), + ) + elif protocol == "legacy-no-run": + connection.execute("DELETE FROM deep_scan_runs") + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + cost = {**BUDGET_COST, "inputTokens": 1500, "estimatedUsd": 0.0075} + result = run_workbench( + state_dir, + "complete-scan", + "--scan-id", + scan_id, + "--cost-json", + json.dumps(cost), + check=not protocol.startswith("future-"), + ) + after = snapshot() + if protocol.startswith("future-"): + assert after == before + assert result["returncode"] != 0 + assert "unsupported" in result["stderr"].lower() + else: + assert result["scan"]["progress"]["status"] == "complete" + assert result["scan"]["cost"] == cost + assert after[1] == before[1] + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT seal_manifest_digest FROM scans").fetchone()[0] + + +@pytest.mark.parametrize("terminal", [False, True]) +@pytest.mark.parametrize("continuation", ["current", "pending", "claimed"]) +def test_budget_completion_checks_continuation_before_draft_writes( + workbench_api, monkeypatch, tmp_path, terminal, continuation +): + monkeypatch.setattr(workbench_test_support, "SCRIPT", workbench_api["__file__"]) + monkeypatch.setenv("CODEX_HOME", str(tmp_path / "home")) + state_dir, _, scan_dir, scan_id, _ = budget_scan_fixture(tmp_path, terminal=terminal) + database = state_dir / "workbench.sqlite3" + with sqlite3.connect(database) as connection: + connection.execute("UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2'") + if continuation != "current": + connection.execute("UPDATE scans SET handoff_status = 'pending'") + if continuation == "claimed": + run_workbench( + state_dir, + "claim-handoff-delivery", + "--scan-id", + scan_id, + "--claim-token", + str(uuid.uuid4()), + ) + + def snapshot(): + with sqlite3.connect(database) as connection: + return list(connection.iterdump()), { + str(path.relative_to(scan_dir)): path.read_bytes() + for path in scan_dir.rglob("*") + if path.is_file() + } + + before = snapshot() + result = complete_budget_scan(state_dir, scan_id, check=continuation == "current") + after = snapshot() + if continuation == "current": + assert result["scan"]["progress"]["status"] == "complete" + else: + assert after == before + assert result["returncode"] != 0 + assert "owned by another continuation" in result["stderr"] + + def complete_budget_scan(state_dir: Path, scan_id: str, *, check: bool = True) -> dict[str, object]: return run_workbench( state_dir, @@ -257,7 +454,7 @@ def test_budget_exhaustion_preserves_unvalidated_discovery_as_deferred_work( assert "User input reaches a SQL statement" in coverage["deferred"][0]["reason"] assert coverage["surfaces"][0]["disposition"] == "needs_follow_up" report = (scan_dir / "report.md").read_text() - assert "No findings were validated before the scan reached its cost limit" in report + assert "No findings are included in this partial report" in report assert "User input reaches a SQL statement" in report @@ -397,10 +594,7 @@ def test_budget_exhaustion_preserves_existing_deferred_work_with_one_trusted_mar "id": "scan-cost-limit", "reason": "Validation was deferred because the scan reached its cost limit.", } - assert ( - "No findings were validated before the scan reached its cost limit" - in (scan_dir / "report.md").read_text() - ) + assert "No findings are included in this partial report" in (scan_dir / "report.md").read_text() def test_budget_exhaustion_preserves_capped_discovery(tmp_path: Path) -> None: @@ -461,17 +655,33 @@ def test_budget_exhaustion_rejects_scan_below_configured_limit(tmp_path: Path) - assert "has not exceeded its configured cost limit" in str(rejected["stderr"]) -def test_budget_exhaustion_rejects_incomplete_discovery(tmp_path: Path) -> None: - state_dir, _, _, scan_id, _ = budget_scan_fixture(tmp_path, terminal=False) - - rejected = complete_budget_scan(state_dir, scan_id, check=False) - - assert rejected["returncode"] != 0 - assert "requires successfully completed Deep Scan discovery" in str(rejected["stderr"]) - assert ( - run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"]["progress"]["status"] - == "running" - ) +@pytest.mark.parametrize("workflow", ["deep-security-scan/v1", "deep-security-scan/v2"]) +def test_budget_exhaustion_during_discovery_preserves_workflow_contract( + tmp_path: Path, workflow: str +) -> None: + state_dir, _, scan_dir, scan_id, ledger = budget_scan_fixture(tmp_path, terminal=False) + original = ledger.read_bytes() + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = ? WHERE scan_id = ?", (workflow, scan_id) + ) + result = complete_budget_scan(state_dir, scan_id, check=False) + if workflow == "deep-security-scan/v1": + assert result["returncode"] != 0 + assert "requires successfully completed Deep Scan discovery" in str(result["stderr"]) + assert ( + run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"]["progress"]["status"] + == "running" + ) + else: + assert result["returncode"] == 0 + saved = run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"] + assert saved["progress"]["status"] == "complete" + assert saved["findings"] == [] + assert saved["cost"] == BUDGET_COST + assert saved["warnings"] == [BUDGET_WARNING] + assert json.loads((scan_dir / "coverage.json").read_text())["completeness"] == "partial" + assert ledger.read_bytes() == original def test_budget_exhaustion_rejects_standard_scan(tmp_path: Path) -> None: @@ -1042,7 +1252,7 @@ def test_workbench_persists_progress_and_indexes_completed_findings(tmp_path: Pa ) } assert tables == EXPECTED_TABLES - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (46,) assert connection.execute("SELECT COUNT(*) FROM findings").fetchone() == (1,) assert connection.execute("SELECT COUNT(*) FROM finding_locations").fetchone() == (1,) diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py index c58a274e2..9d598c4bb 100644 --- a/plugins/codex-security/tests/test_workbench_deep_scan.py +++ b/plugins/codex-security/tests/test_workbench_deep_scan.py @@ -279,7 +279,9 @@ def claim() -> dict[str, object]: return claim_deep_scan_coordinator(state_dir, codex_home, scan_id) with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + migrations = connection.execute( + "SELECT * FROM schema_migrations ORDER BY version" + ).fetchall() assert claim()["deepScan"]["coordinatorGeneration"] == 2 assert claim()["coordinatorDisposition"] == "observing" expire_deep_scan_coordinator(state_dir, scan_id) @@ -311,6 +313,11 @@ def claim() -> dict[str, object]: assert sum(result["coordinatorDisposition"] == "adopted" for result in results) == 1 assert sum(result["coordinatorDisposition"] == "observing" for result in results) == 3 assert {result["deepScan"]["coordinatorGeneration"] for result in results} == {3} + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + assert ( + connection.execute("SELECT * FROM schema_migrations ORDER BY version").fetchall() + == migrations + ) def test_legacy_generation_with_active_worker_observes_grace_then_adopts(tmp_path: Path) -> None: @@ -1879,10 +1886,21 @@ def test_failed_reducer_rebuffers_claimed_inputs_for_same_generation_replacement error="fixture reducer exhausted its attempts", coordinator_generation=2, )["deepScan"] - replayed_workers = {worker["id"]: worker for worker in replayed["workers"]} + assert replayed["workerReceipt"] == failed["workerReceipt"] assert replayed["phase"] == "reducing" - assert replayed["consecutiveErrors"] == counter_before_failure - assert all(replayed_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) + current = run_workbench( + state_dir, + "get-deep-scan", + "--scan-id", + scan_id, + "--thread-id", + "thread-deep-scan", + environment=deep_environment(codex_home), + )["deepScan"] + current_workers = {worker["id"]: worker for worker in current["workers"]} + assert current["phase"] == "reducing" + assert current["consecutiveErrors"] == counter_before_failure + assert all(current_workers[worker]["mergeState"] == "merging" for worker in replacement_inputs) upsert_worker( state_dir, diff --git a/plugins/codex-security/tests/test_workbench_scan_usage.py b/plugins/codex-security/tests/test_workbench_scan_usage.py index 06b7ea425..7711193e6 100644 --- a/plugins/codex-security/tests/test_workbench_scan_usage.py +++ b/plugins/codex-security/tests/test_workbench_scan_usage.py @@ -1,11 +1,14 @@ from __future__ import annotations import json +import os import runpy import sqlite3 +import subprocess import sys import tempfile import uuid +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path @@ -13,6 +16,7 @@ import pytest from workbench_test_support import ( + SCRIPT, create_saved_workspace, initialize_git_repository, mark_deep_coordinator_succeeded, @@ -492,6 +496,32 @@ def test_completion_reports_unavailable_without_fabricating_zero(tmp_path: Path) assert "totalTokens" not in usage +@pytest.mark.parametrize("reported", ["missing", "null-counter", "explicit-zero"]) +def test_completion_distinguishes_missing_token_records_from_zero( + tmp_path: Path, reported: str +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + events = [] + if reported == "explicit-zero": + events.append(_token_event(counted, 0, 0)) + elif reported == "null-counter": + events.append( + _event(counted, "event_msg", {"type": "token_count", "info": None, "rate_limits": None}) + ) + parent = _rollout(tmp_path, "scan-parent", events) + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + if reported == "explicit-zero": + assert usage["coverage"] == "complete" + assert usage["totalTokens"] == 0 + else: + assert usage["coverage"] == "unavailable" + assert "token_usage_unavailable" in usage["warnings"] + assert "token_record_invalid" not in usage["warnings"] + assert "totalTokens" not in usage + + @pytest.mark.skipif(sys.platform != "darwin", reason="macOS system path aliases") @pytest.mark.parametrize("temporary_root", [tempfile.gettempdir(), "/tmp"], ids=["var", "tmp"]) def test_completion_accepts_macos_system_rollout_alias( @@ -605,13 +635,301 @@ def test_completion_counts_deep_sdk_workers_and_descendants(tmp_path: Path) -> N ) usage = _complete_scan(fixture)["scan"]["usage"] assert usage == { - "coverage": "complete", + "coverage": "partial", "source": "codex_rollout", - **_counts(37, 0, 10), - "threadCount": 3, + **_counts(27, 0, 7), + "threadCount": 2, + "missingThreadCount": 1, + "warnings": ["scan_owner_turn_unavailable"], + "modelUsage": [{"model": None, **_counts(27, 0, 7)}], } +@pytest.mark.parametrize( + "worker_home", + [ + "recorded", + "current", + "inherited-sqlite", + "current-prefix", + "recorded-prefix", + "current-unreadable", + "current-mismatched", + "external-sqlite", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + "unavailable", + ], +) +def test_completion_keeps_owner_and_workers_in_their_recorded_homes( + tmp_path: Path, worker_home: str +) -> None: + current_home = tmp_path / "current-home" + copied_rollout = worker_home in { + "current-prefix", + "recorded-prefix", + "current-unreadable", + "current-mismatched", + } + environment = { + "CODEX_HOME": str(current_home), + "CODEX_SQLITE_HOME": str(current_home / "sqlite"), + "CODEX_STATE_DB": str(current_home / "sqlite" / "state_5.sqlite"), + } + owners = { + f"owner-{index}": _rollout( + tmp_path, + f"owner-{index}", + [_event(datetime.now().astimezone(), "turn_context", {"turn_id": "original"})], + ) + for index in (1, 2) + } + _state_graph(environment, owners, []) + + def complete(index: int) -> dict[str, Any]: + root = tmp_path / f"scan-{index}" + target = root / "target" + target.mkdir(parents=True) + selected_home = current_home if worker_home == "current" else root / "original-home" + if worker_home == "external-shared-home": + selected_home = tmp_path / "shared-original-home" + process = subprocess.run( + [ + sys.executable, + "-c", + ( + "import runpy, sys; script = sys.argv.pop(1); " + "runpy.run_path(script)['main'](with_execution_settings=True)" + ), + str(SCRIPT), + "begin-deep-scan", + "--thread-id", + f"owner-{index}", + "--target-path", + str(target), + "--scan-root", + str(root / "scans"), + ], + env={**os.environ, **environment, "CODEX_SECURITY_STATE_DIR": str(root / "state")}, + input=json.dumps( + { + "executionSettings": { + "codexHome": str(selected_home), + "codexPath": sys.executable, + } + } + ), + capture_output=True, + text=True, + timeout=30, + ) + assert process.returncode == 0, process.stderr + deep = json.loads(process.stdout)["deepScan"] + scan_id, scan_dir = deep["scanId"], Path(deep["scanDir"]) + snapshot = scan_dir / "artifacts/deep_discovery/execution-settings.json" + original_bytes = snapshot.read_bytes() + fixture = ScanFixture( + root / "state", + target, + scan_id, + scan_dir, + datetime.fromisoformat(deep["createdAt"]), + environment, + "deep", + ) + counted = fixture.started_at + timedelta(microseconds=1) + with owners[f"owner-{index}"].open("a") as stream: + stream.write(json.dumps(_token_event(counted, index * 10, 2)) + "\n") + stream.write(json.dumps(_event(counted, "turn_context", {"turn_id": "later"})) + "\n") + stream.write(json.dumps(_token_event(counted, 9000, 900)) + "\n") + worker_threads = {} + for kind in ("discovery", "second-discovery"): + thread_id = f"{kind}-{index}" + artifact = scan_dir / "artifacts" / thread_id + artifact.mkdir() + prompt = artifact / "prompt.md" + prompt.write_text("Review the synthetic target.\n") + run_workbench( + fixture.state_dir, + "upsert-deep-scan-worker", + "--scan-id", + scan_id, + "--worker-id", + str(uuid.uuid4()), + "--kind", + "discovery", + "--status", + "running", + "--prompt-path", + str(prompt), + "--artifact-dir", + str(artifact), + "--sdk-thread-id", + thread_id, + environment=environment, + ) + worker_threads[thread_id] = _rollout( + root, + thread_id, + [ + *( + [_event(counted, "turn_context", {"model": "model-alpha"})] + if copied_rollout and kind == "discovery" + else [] + ), + _token_event(counted, index * 20, 3), + _event( + counted, + "turn_context", + { + "turn_id": "resumed", + **( + {"model": "model-beta"} + if copied_rollout and kind == "discovery" + else {} + ), + }, + ), + _token_event(counted, index * 30, 5), + ], + ) + child_id = f"child-{index}" + worker_threads[child_id] = _rollout( + root, + child_id, + [_token_event(counted, index * 7, 1)], + parent_thread_id=f"discovery-{index}", + ) + if worker_home in {"current", "inherited-sqlite"}: + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.executemany( + "INSERT INTO threads VALUES (?, ?)", + [(key, str(path)) for key, path in worker_threads.items()], + ) + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (f"discovery-{index}", child_id) + ) + if worker_home == "inherited-sqlite": + # Earlier launches used A; the resumed process forwards its + # explicit SQLite home C even while workers keep Codex home A. + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + {f"discovery-{index}": worker_threads[f"discovery-{index}"]}, + [], + ) + elif worker_home in { + "recorded", + "current-prefix", + "recorded-prefix", + "current-unreadable", + "current-mismatched", + }: + recorded_threads = dict(worker_threads) + if worker_home != "recorded": + thread_id = f"discovery-{index}" + full = worker_threads[thread_id] + copied = full.with_name(f"copied-{thread_id}.jsonl") + copied.write_bytes(b"\n".join(full.read_bytes().splitlines()[:3]) + b"\n") + if worker_home == "current-unreadable": + copied.write_text("invalid session metadata\n") + elif worker_home == "current-mismatched": + copied.write_text(full.read_text().replace(thread_id, "unrelated-thread")) + current_copy = copied + if worker_home == "recorded-prefix": + recorded_threads[thread_id] = copied + current_copy = full + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads VALUES (?, ?)", (thread_id, str(current_copy)) + ) + _state_graph( + {"CODEX_SQLITE_HOME": str(selected_home)}, + recorded_threads, + [(f"discovery-{index}", child_id)], + ) + elif worker_home in { + "external-sqlite", + "external-shared-home", + "external-missing-copy", + "external-missing-child", + }: + # Native keeps rollouts in its Codex home even when its SQLite + # index lives elsewhere and recovery chooses a different index. + sessions = selected_home / "sessions" / "2026" / "01" / "01" + sessions.mkdir(parents=True, exist_ok=True) + for thread_id, path in worker_threads.items(): + recorded = sessions / f"rollout-{thread_id}.jsonl" + path.rename(recorded) + worker_threads[thread_id] = recorded + _state_graph( + {"CODEX_SQLITE_HOME": str(root / "original-external-sqlite")}, + worker_threads, + [(f"discovery-{index}", child_id)], + ) + if worker_home in {"external-missing-copy", "external-missing-child"}: + first_id = f"discovery-{index}" + with sqlite3.connect(environment["CODEX_STATE_DB"]) as connection: + connection.execute( + "INSERT INTO threads VALUES (?, ?)", + ( + first_id, + str(root / "missing-copy.jsonl") + if worker_home == "external-missing-copy" + else str(worker_threads[first_id]), + ), + ) + if worker_home == "external-missing-child": + connection.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?)", (first_id, child_id) + ) + worker_threads[child_id].unlink() + result = _complete_scan(fixture)["scan"]["usage"] + assert snapshot.read_bytes() == original_bytes + return result + + # Both completions share current home B; each scan retains its own worker home A. + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(complete, (1, 2))) + for index, usage in enumerate(results, 1): + if worker_home == "unavailable": + assert usage["coverage"] == "partial" + assert usage["inputTokens"] == index * 10 + assert usage["outputTokens"] == 2 + assert usage["threadCount"] == 1 + assert usage["missingThreadCount"] == 2 + elif worker_home == "external-missing-child": + assert usage == { + "coverage": "partial", + "source": "codex_rollout", + **_counts(index * 70, 0, 12), + "threadCount": 3, + "missingThreadCount": 1, + "warnings": [ + "codex_state_unavailable", + "rollout_unavailable", + "scan_root_unavailable", + ], + "modelUsage": [{"model": None, **_counts(index * 70, 0, 12)}], + } + else: + assert usage == { + "coverage": "complete", + "source": "codex_rollout", + **_counts(index * 77, 0, 13), + "threadCount": 4, + "modelUsage": ( + [ + {"model": None, **_counts(index * 47, 0, 8)}, + {"model": "model-alpha", **_counts(index * 20, 0, 3)}, + {"model": "model-beta", **_counts(index * 10, 0, 2)}, + ] + if copied_rollout + else [{"model": None, **_counts(index * 77, 0, 13)}] + ), + } + + def test_completion_preserves_explicit_legacy_cost(tmp_path: Path) -> None: fixture = _start_scan(tmp_path) counted = fixture.started_at + timedelta(microseconds=1) @@ -684,3 +1002,273 @@ def test_failed_scan_preserves_legacy_failure_behavior(tmp_path: Path) -> None: )["scan"] assert failed["progress"]["status"] == "failed" assert "usage" not in failed + + +def test_rollout_usage_reconciles_stale_cumulative_events_and_models( + tmp_path: Path, workbench_api +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + events = [ + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-5.6-sol"}), + _token_event(start, 100, 10), + _token_event(start, 60, 6), + _event(start, "turn_context", {"turn_id": "own-turn", "model": "gpt-6-astra"}), + _token_event(start, 200, 20), + ] + rollout = _rollout(tmp_path, "worker", events) + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + ) + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + models = {} + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("worker", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + assert counts == _counts(200, 0, 20) + assert warnings == {"token_counter_regressed"} + assert models == {"gpt-5.6-sol": _counts(100, 0, 10), "gpt-6-astra": _counts(100, 0, 10)} + + +def test_shared_parent_usage_requires_original_turn_and_scan_interval( + tmp_path: Path, workbench_api +) -> None: + usage_reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + end = start + timedelta(seconds=5) + events = [ + _event( + start - timedelta(seconds=1), + "turn_context", + {"turn_id": "prior", "model": "gpt-5.6-sol"}, + ), + _token_event(start - timedelta(seconds=1), 100, 10), + _event(start, "turn_context", {"turn_id": "scan-turn", "model": "gpt-6-astra"}), + _token_event(start, 110, 12), + _event(start, "turn_context", {"turn_id": "unrelated", "model": "gpt-5.6-sol"}), + _token_event(start, 910, 92), + _event( + end + timedelta(seconds=1), + "turn_context", + {"turn_id": "scan-turn", "model": "gpt-6-astra"}, + ), + _token_event(end + timedelta(seconds=1), 1000, 100), + ] + models = {} + counts, warnings = usage_reader._read_rollout_usage( + usage_reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=end, + owner_turn_id="scan-turn", + model_usage=models, + ) + assert counts == _counts(10, 0, 2) + assert warnings == set() + assert models == {"gpt-6-astra": _counts(10, 0, 2)} + + +@pytest.mark.parametrize( + "counter_info,receipt_state,expected_warning", + [ + (None, "complete", None), + ({}, "complete", "token_record_invalid"), + ({"total_token_usage": {"input_tokens": -1}}, "complete", "token_record_invalid"), + (None, "missing-response", "token_receipts_incomplete"), + (None, "incomplete-line", "rollout_record_incomplete"), + (None, "invalid-timestamp", "token_record_invalid"), + ], + ids=[ + "null-counter", + "empty-info", + "malformed-usage", + "missing-response", + "incomplete-line", + "invalid-timestamp", + ], +) +def test_completion_handles_no_usage_counter_without_hiding_incomplete_receipts( + tmp_path: Path, counter_info: Any, receipt_state: str, expected_warning: str | None +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + tokens = dict(input_tokens=100, cached_input_tokens=20, output_tokens=10, total_tokens=110) + response = _event( + counted, + "token_usage_record", + dict( + response_id="response-one", + thread_id="scan-parent", + model="gpt-5.6-sol", + usage=tokens, + thread_token_usage=( + {**tokens, "input_tokens": 150, "total_tokens": 160} + if receipt_state == "missing-response" + else tokens + ), + ), + ) + counter = _event( + counted, + "event_msg", + {"type": "token_count", "info": counter_info, "rate_limits": None}, + ) + events = [counter, response, counter] + if receipt_state == "invalid-timestamp": + events.append( + { + **response, + "timestamp": None, + "payload": {**response["payload"], "response_id": "response-two"}, + } + ) + parent = _rollout(tmp_path, "scan-parent", events) + if receipt_state == "incomplete-line": + with parent.open("a") as stream: + stream.write('{"type":"token_usage_record"') + _state_graph(fixture.environment, {"scan-parent": parent}, []) + usage = _complete_scan(fixture)["scan"]["usage"] + assert usage["totalTokens"] == 110 + assert usage["modelUsage"] == [{"model": "gpt-5.6-sol", **_counts(100, 20, 10)}] + assert usage["coverage"] == ("partial" if expected_warning else "complete") + assert usage.get("warnings", []) == ([expected_warning] if expected_warning else []) + + +@pytest.mark.parametrize("counters", [False, True]) +def test_response_receipts_count_compaction_once_across_resets( + tmp_path: Path, workbench_api, counters: bool +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def usage(input_tokens, cached, output): + return dict( + input_tokens=input_tokens, + cached_input_tokens=cached, + cache_write_input_tokens=0, + output_tokens=output, + reasoning_output_tokens=0, + total_tokens=input_tokens + output, + ) + + def receipt(response, count, cumulative, model="gpt-5.6-sol", turn="scan-turn", second=1): + return _event( + start + timedelta(seconds=second), + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + turn_id=turn, + model=model, + usage=count, + thread_token_usage=cumulative, + ), + ) + + first = receipt("first", usage(100, 80, 10), usage(100, 80, 10)) + compact = receipt("compaction", usage(50, 40, 5), usage(150, 120, 15), model="gpt-6-astra") + second = receipt("second", usage(120, 90, 12), usage(120, 90, 12)) + events = [ + first, + *([_token_event(start + timedelta(seconds=1), 100, 10)] if counters else []), + compact, + _event(start + timedelta(seconds=1), "compacted", {"message": "Synthetic summary"}), + compact, + second, + *([_token_event(start + timedelta(seconds=1), 220, 22)] if counters else []), + first, + receipt("other", usage(900, 0, 0), usage(900, 0, 0), turn="other-turn"), + receipt("post", usage(800, 0, 0), usage(1700, 0, 0), second=11), + ] + models = {} + total, warnings = reader._read_rollout_usage( + reader.RolloutSession("parent", None, _rollout(tmp_path, "parent", events)), + started_at=start, + completed_at=start + timedelta(seconds=10), + owner_turn_id="scan-turn", + model_usage=models, + ) + assert total == _counts(270, 210, 27) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(220, 170, 22), "gpt-6-astra": _counts(50, 40, 5)} + + +def test_delayed_response_receipt_resolves_missing_cumulative_usage( + tmp_path: Path, workbench_api +) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout(tmp_path, "parent", [receipt("first", 100, 100), receipt("third", 50, 180)]) + session = reader.RolloutSession("parent", None, rollout) + total, warnings = reader._read_rollout_usage(session, started_at=start, completed_at=None) + assert total == _counts(150, 0, 0) + assert warnings == {"token_receipts_incomplete"} + with rollout.open("a") as source: + source.write(json.dumps(receipt("second", 30, 130)) + "\n") + total, warnings = reader._read_rollout_usage(session, started_at=start, completed_at=None) + assert total == _counts(180, 0, 0) + assert warnings == set() + + +def test_exact_receipts_replace_overlapping_legacy_counter(tmp_path: Path, workbench_api) -> None: + reader = sys.modules["workbench_scan_usage"] + start = datetime.fromisoformat("2026-01-01T00:00:00+00:00") + + def receipt(response, tokens, cumulative): + def usage(value): + return dict(input_tokens=value, output_tokens=0, total_tokens=value) + + return _event( + start, + "token_usage_record", + dict( + response_id=response, + thread_id="parent", + model="gpt-5.6-sol", + usage=usage(tokens), + thread_token_usage=usage(cumulative), + ), + ) + + rollout = _rollout( + tmp_path, + "parent", + [ + _token_event(start, 100, 0), + receipt("new", 10, 110), + receipt("old", 100, 100), + _token_event(start, 10, 0), + ], + ) + models = {} + total, warnings = reader._read_rollout_usage( + reader.RolloutSession("parent", None, rollout), + started_at=start, + completed_at=None, + model_usage=models, + ) + assert total == _counts(110, 0, 0) + assert warnings == set() + assert models == {"gpt-5.6-sol": _counts(110, 0, 0)} diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 679ee77c1..f551a3964 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -405,7 +405,7 @@ def test_workbench_serializes_concurrent_first_run_migrations(tmp_path: Path) -> {"databasePath": str(state_dir / "workbench.sqlite3")}, ] with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (46,) @pytest.mark.parametrize("previous_history", ["main", "comparison-preview"]) @@ -867,6 +867,11 @@ def test_workbench_creates_single_final_schema(tmp_path: Path) -> None: (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (44, "preserve original deep scan discovery context"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), ] assert {row[1] for row in connection.execute("PRAGMA table_info(workspaces)")} >= { "diff_target_kind", @@ -969,7 +974,7 @@ def test_workbench_upgrades_preexisting_database(tmp_path: Path) -> None: connection.execute("ALTER TABLE scans DROP COLUMN handoff_claim_token") run_workbench(state_dir, "database-info") with sqlite3.connect(database) as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (48,) assert {row[1] for row in connection.execute("PRAGMA table_info(scans)")} >= { "handoff_claimed_at", "handoff_claim_token", @@ -1996,6 +2001,11 @@ def test_workbench_upgrades_released_database_schema(tmp_path: Path) -> None: (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (44, "preserve original deep scan discovery context"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), ] assert "capability_preflight_json" in { row[1] for row in connection.execute("PRAGMA table_info(workspaces)") @@ -2079,6 +2089,11 @@ def test_workbench_upgrades_pre_release_phase_progress_migration(tmp_path: Path) (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (44, "preserve original deep scan discovery context"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), ] assert "continuation_thread_id" in { row[1] for row in connection.execute("PRAGMA table_info(scans)") @@ -2170,6 +2185,11 @@ def test_workbench_upgrades_pre_release_preflight_progress_migration(tmp_path: P (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (44, "preserve original deep scan discovery context"), + (45, "retain deep scan attempts and exact merge inputs"), + (46, "persist selected deep scan finalization input"), + (47, "freeze stopped scan checkpoint selections"), + (48, "bind original deep scan parent usage turn"), ] assert "continuation_thread_id" in { row[1] for row in connection.execute("PRAGMA table_info(scans)") diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py index de2b5ec22..a88e1680b 100644 --- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py +++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py @@ -47,8 +47,10 @@ def test_stopped_deep_scan_ignores_late_worker_checkpoints_without_reducer( # The latest incomplete attempt need not be parseable for a saved checkpoint to survive. result_path.write_text("{incomplete") with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + # This is new incomplete work, not a rewrite of the accepted attempt. connection.execute( - "UPDATE deep_scan_workers SET status = 'running' WHERE id = ?", (worker_id,) + "UPDATE deep_scan_workers SET status = 'running', attempt = 2 WHERE id = ?", + (worker_id,), ) environment = {"CODEX_HOME": str(codex_home)} if termination == "canceled": @@ -366,6 +368,9 @@ def test_explicit_recovery_preserves_sealed_parent_with_empty_source_map( state_dir, codex_home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.unlink() + # Remove the immutable accepted copy too, leaving no recoverable worker source. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() contract_dir = tmp_path / "contract" contract_dir.mkdir() scripts_dir = Path(__file__).resolve().parents[1] / "scripts" @@ -1012,6 +1017,9 @@ def test_canceled_scan_reports_noop_coordinator_publication(tmp_path: Path) -> N state_dir, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path) _, result_path = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id) result_path.write_text("{incomplete") + # A valid immutable copy would let publication recover despite this corruption. + for checkpoint in (result_path.parent / "checkpoints").glob("*.json"): + checkpoint.unlink() scripts_dir = Path(__file__).resolve().parents[1] / "scripts" wrapper = tmp_path / "fail_before_canceled_sources_are_frozen.py" @@ -1395,6 +1403,7 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft(tmp_path: state_dir, codex_home, scan_dir, scan_id, worker_id, result_path ) reduced = json.loads(reducer_path.read_text()) + accepted_summary = reduced["findings"][0]["summary"] reduced["findings"][0]["summary"] = ( "The reducer retained additional independently reviewed evidence." ) @@ -1411,7 +1420,8 @@ def test_failure_preserves_last_committed_reducer_without_parent_draft(tmp_path: failed = run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"] assert failed["progress"]["status"] == "failed" assert failed["findingCount"] == 1 - assert failed["findings"][0]["summary"] == reduced["findings"][0]["summary"] + assert failed["findings"][0]["summary"] == accepted_summary + assert json.loads(reducer_path.read_text()) == reduced def test_stopped_rejection_recovers_malformed_parent_surfaces(tmp_path: Path) -> None: @@ -1584,7 +1594,7 @@ def test_complete_worker_supersedes_obsolete_checkpoint_coverage(tmp_path: Path) }, } checkpoints = result_path.parent / "checkpoints" - checkpoints.mkdir() + checkpoints.mkdir(exist_ok=True) (checkpoints / ("0" * 64 + ".json")).write_text(json.dumps(checkpoint)) run_workbench( @@ -1826,7 +1836,7 @@ def test_recovery_selects_strongest_same_finding_checkpoint(tmp_path: Path) -> N strong["confidence"]["level"] = "high" strong["summary"] = "Later strong checkpoint evidence." checkpoint_dir = result_path.parent / "checkpoints" - checkpoint_dir.mkdir() + checkpoint_dir.mkdir(exist_ok=True) for name, finding in (("0" * 64, weak), ("f" * 64, strong)): (checkpoint_dir / f"{name}.json").write_text( json.dumps( diff --git a/plugins/codex-security/tests/workbench_test_support.py b/plugins/codex-security/tests/workbench_test_support.py index dae12b77e..4a9017c1f 100644 --- a/plugins/codex-security/tests/workbench_test_support.py +++ b/plugins/codex-security/tests/workbench_test_support.py @@ -212,7 +212,7 @@ def create_saved_git_workspace(state_dir: Path, target: Path) -> dict[str, objec def mark_deep_coordinator_succeeded(state_dir: Path, scan_id: str, scan_dir: Path) -> Path: manifest = scan_dir / "artifacts" / "deep_discovery" / "coordinator-manifest.json" - manifest.parent.mkdir(parents=True) + manifest.parent.mkdir(parents=True, exist_ok=True) manifest.write_text('{"status":"succeeded"}\n') with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: connection.execute( diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index bd86a0fc3..67ee33cfd 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -799,6 +799,12 @@ four workers. Unknown keys are rejected. `max_time_hours` accepts positive values up to 96, including fractional hours. At the deadline, discovery stops; the scan combines and returns completed findings. +New Deep scans save the accepted aggregate before publishing the result. If +publication is interrupted, recovering the same scan reuses that aggregate and +its original stop reason without another discovery or reducer run. Explicit +cancellation and cost stops retain their stopped or partial-result behavior. +Scans created by earlier versions keep their original workflow when resumed. + `scan --workers` controls discovery workers within one deep scan; `bulk-scan --workers` controls how many repositories are scanned concurrently. @@ -941,6 +947,14 @@ including cache reads and writes. They exclude long-context and other processing tier adjustments, fees, and surcharges. GPT-5.5 and GPT-6 Astra are supported; models without known prices show an unavailable estimate. +Deep Scan accounting includes failed, replaced, and canceled worker attempts and +their descendants once. Shared conversation usage is limited to the original +scan turn and scan interval. Missing usage remains unavailable or partial; +reported zero remains zero. When sessions use different models, `cost.modelCosts` +records each model's tokens, estimate, and pricing basis, and `estimatedUsd` sums +those estimates. A partial estimate has `cost.coverage: "partial"`; missing model +attribution leaves the estimate unavailable until usage can be reconciled. + For compatibility, `cacheWriteInputTokens` remains the reported token subtotal. `cacheWriteInputTokensReported: false` means at least one included usage record did not report cache writes. Raw usage uses `cache_write_input_tokens_reported`. diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs index f78271374..66850779f 100644 --- a/sdk/typescript/scripts/check-package.mjs +++ b/sdk/typescript/scripts/check-package.mjs @@ -166,15 +166,19 @@ const allowedRoot = new Set([ ]); const distFiles = new Set( [ + "accepted-audit", "api", "auth", "bulk-scan-discovery", "cli", + "cli-scan-logs-json", "classify-severity", "classify-scan-severity", "severity-store", "cloud-publish", "codex-prompt", + "codex-session", + "preflight-config", "component-plan", "component-scan", "config", @@ -183,6 +187,7 @@ const distFiles = new Set( "cost", "cost-model", "custom-validation", + "deep-scan-finalization", "custom-validation-prompt", "custom-publish", "deep-progress", @@ -210,6 +215,7 @@ const distFiles = new Set( "publication-events", "publication-store", "publish", + "reasoning-summary", "result", "runtime", "scan-activity", @@ -327,7 +333,17 @@ assertExpectedGitHead( const internalMarker = /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?:[^a-z0-9_-]|$)|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?:^|[^a-z0-9_-])go\/[a-z0-9_-]+)/iu; -const payloads = [archiveBytes.toString("utf8")]; +// Scan compressed assets after decoding them below. Their binary bytes can +// coincidentally match text references; keep scanning every tar header and all +// other entry contents. +const readableArchive = Buffer.from(archiveBytes); +for (const file of files) { + if (!/\.br(?:\.part-[0-9]+)?$/iu.test(file)) continue; + const bytes = archiveFile(file); + const start = bytes.byteOffset - archiveBytes.byteOffset; + readableArchive.fill(0, start, start + bytes.byteLength); +} +const payloads = [readableArchive.toString("utf8")]; const compressedFiles = [...files].filter((file) => /\.br$/iu.test(file)); const compressedParts = new Map(); for (const file of files) { diff --git a/sdk/typescript/scripts/ci-test-durations.json b/sdk/typescript/scripts/ci-test-durations.json index 40c7b63d3..431742962 100644 --- a/sdk/typescript/scripts/ci-test-durations.json +++ b/sdk/typescript/scripts/ci-test-durations.json @@ -2,7 +2,9 @@ "sources": [ "https://github.com/openai/codex-security/actions/runs/33215429477", "https://github.com/openai/codex-security/actions/runs/33227512538", - "https://github.com/openai/codex-security/actions/runs/33233690024" + "https://github.com/openai/codex-security/actions/runs/33233690024", + "https://github.com/openai/codex-security/actions/runs/34813432568", + "https://github.com/openai/codex-security/actions/runs/34822813350" ], "unix": { "api-attribution-concurrency.test.ts": 0.372, @@ -158,6 +160,7 @@ "cost.test.ts": 1.574, "custom-publish.test.ts": 0.582, "custom-validation.test.ts": 20.415, + "deep-finalization.test.ts": 212.524, "deep-progress.test.ts": 0.007, "deep-scan-parent-denials.test.ts": 0.013, "deep-scan-reducer-recovery.test.ts": 0.527, @@ -209,6 +212,7 @@ "scan-history-renderer.test.ts": 0.005, "scan-logs.test.ts": 0.139, "scan-recovery.test.ts": 62.302, + "scan-resume.test.ts": 207.628, "skeleton.test.ts": 0.03, "stopped-scan-results.test.ts": 15.814, "targets-large-output.test.ts": 11.48, diff --git a/sdk/typescript/scripts/fixtures/package-deep-codex.mjs b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs new file mode 100644 index 000000000..4d645dfb9 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-codex.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { appendFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { createInterface } from "node:readline"; +import { startRpc } from "./package-rpc.mjs"; + +try { + await run(); + process.exit(0); +} catch (error) { + await trace({ phase: "fixture-error", error: error.stack }); + console.error(error); + process.exit(1); +} + +async function trace(event) { + await appendFile( + process.env.PACKAGE_DEEP_TRACE, + `${JSON.stringify(event)}\n`, + ); +} + +async function run() { + const args = process.argv.slice(2); + if (args.includes("app-server")) { + await trace({ phase: "preflight", args }); + for await (const line of createInterface({ input: process.stdin })) { + const message = JSON.parse(line); + if (message.id === undefined) continue; + let result; + switch (message.method) { + case "initialize": + result = { userAgent: "package-fixture" }; + break; + case "config/read": + result = { + config: { + default_permissions: "codex_security_deep_scan_worker", + permissions: { + codex_security_deep_scan_worker: { + extends: ":read-only", + filesystem: { ":root": "read" }, + network: { enabled: false }, + }, + }, + }, + origins: {}, + layers: null, + }; + break; + case "permissionProfile/list": + result = { + data: [ + { + id: "codex_security_deep_scan_worker", + description: null, + allowed: true, + }, + ], + nextCursor: null, + }; + break; + case "account/read": + result = { account: null, requiresOpenaiAuth: true }; + break; + default: + throw new Error(`Unexpected preflight method: ${message.method}`); + } + console.log(JSON.stringify({ jsonrpc: "2.0", id: message.id, result })); + } + return; + } + let prompt = ""; + for await (const chunk of process.stdin) prompt += chunk; + const config = {}; + for (let index = 0; index < args.length; index++) { + if (args[index] !== "-c" && args[index] !== "--config") continue; + const setting = args[++index]; + const equals = setting.indexOf("="); + config[setting.slice(0, equals)] = setting.slice(equals + 1); + } + const prefix = "mcp_servers.cs_artifacts."; + const env = Object.fromEntries( + Object.entries(config) + .filter(([name]) => name.startsWith(`${prefix}env.`)) + .map(([name, value]) => [ + name.slice(`${prefix}env.`.length), + JSON.parse(value), + ]), + ); + const root = env.CODEX_SECURITY_ARTIFACT_ROOT; + assert.ok(root, "The real worker must supply its bound artifact root."); + assert.equal(config["mcp_servers.codex-security.enabled"], "false"); + const layout = env.CODEX_SECURITY_ARTIFACT_LAYOUT; + const threadId = `package-${layout}-${basename(root)}-${basename(join(root, ".."))}`; + console.log(JSON.stringify({ type: "thread.started", thread_id: threadId })); + if ( + layout === "worker" && + basename(join(root, "..")) === "discovery-0002" && + process.env.PACKAGE_DEEP_HOLD && + existsSync(process.env.PACKAGE_DEEP_HOLD) + ) { + await trace({ phase: "held", scanId: env.CODEX_SECURITY_SCAN_ID }); + await new Promise(() => setInterval(() => {}, 1_000)); + } + const server = await startRpc( + JSON.parse(config[`${prefix}command`]), + JSON.parse(config[`${prefix}args`]), + { cwd: root, env: { ...process.env, ...env } }, + ); + let complete = true; + try { + if (layout === "worker") { + const draft = { + scanId: env.CODEX_SECURITY_SCAN_ID, + findings: [], + coverage: { + completeness: "complete", + surfaces: [], + explicitExclusions: [], + deferred: [], + }, + }; + const marker = process.env.PACKAGE_DEEP_EMPTY_ONCE; + if (marker && !existsSync(marker)) { + await writeFile(marker, "process completed without a final artifact"); + complete = false; + } else { + await server.call("record_codex_security_scan_draft", { + ...draft, + complete: false, + }); + await server.call("record_codex_security_scan_draft", { + ...draft, + complete: true, + }); + } + } else { + assert.equal(layout, "reducer"); + const inputs = await server.call( + "get_codex_security_deep_reducer_inputs", + {}, + ); + assert.ok(inputs.discoveries.length > 0); + await server.call("record_codex_security_deep_reduction", { + scanId: env.CODEX_SECURITY_SCAN_ID, + findings: [], + }); + } + await trace({ + phase: layout, + complete, + resumed: args.includes("resume"), + scanId: env.CODEX_SECURITY_SCAN_ID, + home: process.env.CODEX_HOME, + hasApiKey: process.env.CODEX_API_KEY === "synthetic-package-deep-key", + root, + args, + }); + console.log( + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 1, + cached_input_tokens: 0, + output_tokens: 1, + }, + }), + ); + } finally { + await server.close(); + } +} diff --git a/sdk/typescript/scripts/fixtures/package-deep-scan.mjs b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs new file mode 100644 index 000000000..702704903 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-scan.mjs @@ -0,0 +1,595 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + chmod, + copyFile, + cp, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { startRpc } from "./package-rpc.mjs"; +import { packageSmokeTimeouts } from "../package-smoke-timeouts.mjs"; + +const installedRoot = await realpath(process.argv[2]); +const root = await realpath( + await mkdtemp(join(tmpdir(), "package deep % fixture-")), +); +const installedPlugin = join(installedRoot, "_bundled_plugin"); +try { + const detachedPlugin = join(root, "standalone plugin %", "codex-security"); + await cp(installedPlugin, detachedPlugin, { recursive: true }); + // Assert physical independence instead of merely changing the working directory. + for (let ancestor = detachedPlugin; ; ancestor = dirname(ancestor)) { + for (const dependency of ["node_modules", join("sdk", "typescript")]) { + await assert.rejects(stat(join(ancestor, dependency)), { + code: "ENOENT", + }); + } + if (ancestor === dirname(ancestor)) break; + } + for (const name of [ + "package-deep-codex.mjs", + "package-rpc.mjs", + "package-deep-spawn.mjs", + ]) { + await copyFile(new URL(name, import.meta.url), join(root, name)); + } + const executable = join( + root, + process.platform === "win32" + ? "package-codex.exe" + : "package-deep-codex.mjs", + ); + if (process.platform === "win32") + await copyFile(process.execPath, executable); + await chmod(executable, 0o700); + + await runInstalledSdk(installedPlugin, executable); + await runDetachedPlugin(detachedPlugin, executable); + console.log( + "Validated installed SDK and detached plugin: real Deep processes, bound artifact tools, checkpoints, reducer acceptance, restart before finalization, and sealed results.", + ); +} catch (error) { + for (const name of ["installed", "detached"]) { + try { + error.message += `\n${await readFile(join(root, name, "executions.jsonl"), "utf8")}`; + } catch (readError) { + if (readError.code !== "ENOENT") throw readError; + } + } + throw error; +} finally { + await rm(root, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 100, + }); +} + +async function fixture(name, pluginRoot, executable) { + const directory = join(root, name); + const target = join(directory, "target with spaces %"); + const home = join(directory, "home"); + await mkdir(target, { recursive: true }); + await mkdir(join(home, "codex-security"), { recursive: true }); + await writeFile( + join(target, "fixture.py"), + "print('synthetic package fixture')\n", + ); + await writeFile( + join(home, "codex-security", "config.toml"), + "[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = 1\nmax_discovery_runs = 2\n", + ); + const env = Object.fromEntries( + [ + "PATH", + "Path", + "SystemRoot", + "WINDIR", + "ComSpec", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + ] + .filter((key) => process.env[key] !== undefined) + .map((key) => [key, process.env[key]]), + ); + Object.assign(env, { + HOME: home, + USERPROFILE: home, + CODEX_HOME: home, + CODEX_CLI_PATH: executable, + CODEX_SECURITY_PLUGIN_ROOT: pluginRoot, + CODEX_SECURITY_STATE_DIR: join(directory, "state"), + CODEX_SECURITY_SCAN_ROOT: join(directory, "scans"), + PYTHON: process.env.PYTHON || "python3", + OPENAI_API_KEY: "synthetic-package-deep-key", + ...(process.platform === "win32" + ? { + PACKAGE_DEEP_EXECUTABLE: executable, + NODE_OPTIONS: `--import=${pathToFileURL(join(root, "package-deep-spawn.mjs")).href}`, + } + : {}), + PACKAGE_DEEP_TRACE: join(directory, "executions.jsonl"), + }); + return { directory, target, home, env, pluginRoot }; +} + +function metadata(f, owner) { + return { + "openai/threadId": owner, + "codex/sandbox-state-meta": { + permissionProfile: { + type: "managed", + file_system: { + type: "restricted", + entries: [ + { + path: { type: "special", value: { kind: "root" } }, + access: "read", + }, + ], + }, + network: "restricted", + }, + sandboxCwd: pathToFileURL(f.target).href, + }, + "x-codex-turn-metadata": { model: "gpt-5.5", reasoning_effort: "high" }, + }; +} + +function server(f, env = f.env) { + return startRpc( + process.execPath, + [join(f.pluginRoot, "mcp", "server.mjs"), "--stdio"], + { + cwd: f.target, + env, + requestTimeoutMs: packageSmokeTimeouts().commandTimeoutMs, + }, + ); +} + +async function runDetachedPlugin(pluginRoot, executable) { + const f = await fixture("detached", pluginRoot, executable); + const owner = "package-detached-owner"; + f.env.PACKAGE_DEEP_HOLD = join(f.directory, "hold-second-worker"); + await writeFile(f.env.PACKAGE_DEEP_HOLD, "hold"); + let rpc = await server(f); + let scanId; + let scanDir; + let partial; + const handoffClaimToken = randomUUID(); + try { + const opened = await rpc.call( + "open_codex_security_workspace", + { + targetPath: f.target, + scope: ".", + mode: "deep", + }, + metadata(f, owner), + ); + const sessionId = opened.workspace.id; + await rpc.call( + "submit_codex_security_setup", + { + sessionId, + targetPath: f.target, + scope: ".", + mode: "deep", + }, + metadata(f, owner), + ); + const started = await rpc.call( + "start_codex_security_scan", + { sessionId }, + metadata(f, owner), + ); + ({ scanId, scanDir } = started.workspace.results); + await rpc.call( + "claim_codex_security_scan_handoff_delivery", + { + scanId, + claimToken: handoffClaimToken, + }, + metadata(f, owner), + ); + await rpc.call( + "attach_codex_security_scan_continuation_thread", + { + scanId, + claimToken: handoffClaimToken, + threadId: owner, + }, + metadata(f, owner), + ); + const pending = rpc + .call( + "start_codex_security_deep_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ) + .catch((error) => error); + const deadline = Date.now() + 30_000; + while (!(await readExecutions(f)).some((entry) => entry.phase === "held")) { + assert.ok( + Date.now() < deadline, + "Second worker did not reach the interruption boundary.", + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + partial = ( + await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]) + ).deepScan; + assert.equal( + partial.workers.filter( + (worker) => + worker.kind === "discovery" && worker.status === "succeeded", + ).length, + 1, + ); + await rpc.close(); + await pending; + } finally { + await rpc.close(); + } + // Simulate an expired owner lease without waiting for wall-clock expiry. Keep + // the real stored workers/results and use the production recovery path. + await rm( + join( + scanDir, + "artifacts", + "deep_discovery", + `coordinator-heartbeat-${partial.coordinatorGeneration}.json`, + ), + { force: true }, + ); + await promisify(execFile)( + f.env.PYTHON, + [ + "-c", + "import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); c.execute('UPDATE deep_scan_runs SET updated_at = ? WHERE scan_id = ?', ('2000-01-01T00:00:00Z',sys.argv[2])); c.commit()", + join(f.env.CODEX_SECURITY_STATE_DIR, "workbench.sqlite3"), + scanId, + ], + { env: f.env }, + ); + await rm(f.env.PACKAGE_DEEP_HOLD); + rpc = await server(f); + try { + const result = await rpc.call( + "start_codex_security_deep_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + await assertDraft(result.manifestPath); + const recovered = ( + await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]) + ).deepScan; + assert.equal( + recovered.coordinatorGeneration, + partial.coordinatorGeneration + 1, + ); + assert.equal(recovered.dispatchedCount, 2); + const retained = partial.workers.find( + (worker) => worker.status === "succeeded", + ); + assert.equal( + recovered.workers.find((worker) => worker.id === retained.id).status, + "succeeded", + ); + } finally { + await rpc.close(); + } + // Publication uses the saved aggregate after the recovered executor exits too. + rpc = await server(f); + try { + await rpc.call( + "complete_codex_security_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + const completed = await rpc.call( + "get_codex_security_completed_scan", + { scanId, handoffClaimToken }, + metadata(f, owner), + ); + assert.equal(completed.manifest.scan.id, scanId); + assert.equal(completed.manifest.scan.status, "completed"); + assert.ok(completed.manifest.scan.sealedAt); + } finally { + await rpc.close(); + } + await assertSavedState(f, scanId, owner); + await assertExecutions(f, scanId, 4); +} + +async function workbench(f, args) { + const { stdout } = await promisify(execFile)( + f.env.PYTHON, + [join(f.pluginRoot, "scripts", "workbench_db.py"), ...args], + { + env: f.env, + cwd: f.target, + maxBuffer: 4 * 1024 * 1024, + }, + ); + return JSON.parse(stdout); +} + +async function runInstalledSdk(pluginRoot, executable) { + const f = await fixture("installed", pluginRoot, executable); + f.env.PACKAGE_DEEP_EMPTY_ONCE = join( + f.directory, + "missing-result-completion", + ); + const sdk = await import( + pathToFileURL(join(installedRoot, "dist", "index.js")).href + ); + const manifest = JSON.parse( + await readFile(join(pluginRoot, ".codex-plugin", "plugin.json"), "utf8"), + ); + const owner = "package-sdk-owner"; + const postScanPrompt = "Explain the completed synthetic scan."; + const prompts = []; + let threadCount = 0; + let manifestBeforeFollowUp; + let scanId; + const client = new sdk.CodexSecurity( + { pythonPath: f.env.PYTHON }, + { + environment: f.env, + prepareRuntime: async () => ({ + codexHome: f.home, + environment: f.env, + credentialsAvailable: true, + plugin: { + pluginRoot, + marketplaceRoot: pluginRoot, + installedRoot: pluginRoot, + marketplaceName: "codex-security-sdk", + name: manifest.name, + version: manifest.version, + }, + }), + // Replace only the parent model's tool choice. The installed SDK registers + // and finalizes the scan; the packaged MCP runs the real Deep lifecycle. + createCodex({ env, apiKey }) { + return { + startThread() { + threadCount += 1; + return { + id: owner, + async runStreamed(prompt) { + prompts.push(prompt); + return { + events: (async function* () { + yield { type: "thread.started", thread_id: owner }; + if (prompts.length > 1) { + assert.equal(prompt, postScanPrompt); + manifestBeforeFollowUp = await readFile( + join(env.CODEX_SECURITY_SCAN_DIR, "scan-manifest.json"), + "utf8", + ); + const completed = JSON.parse(manifestBeforeFollowUp); + assert.equal(completed.scan.status, "completed"); + assert.ok(completed.scan.sealedAt); + yield { + type: "turn.completed", + usage: { + input_tokens: 100_000, + cached_input_tokens: 0, + output_tokens: 100_000, + }, + }; + return; + } + scanId = env.CODEX_SECURITY_SCAN_ID; + // The pinned SDK maps its apiKey option to this child variable. + const rpc = await server(f, { + ...env, + ...(apiKey ? { CODEX_API_KEY: apiKey } : {}), + }); + try { + const result = await rpc.call( + "start_codex_security_deep_scan", + { scanId }, + metadata(f, owner), + ); + await assertDraft(result.manifestPath); + } finally { + await rpc.close(); + } + yield { + type: "turn.completed", + usage: { + input_tokens: 1, + cached_input_tokens: 0, + output_tokens: 1, + }, + }; + })(), + }; + }, + }; + }, + }; + }, + }, + ); + try { + const result = await client.run(f.target, { + mode: "deep", + auth: "api-key", + workers: 1, + subagents: 0, + maxDiscoveryRuns: 2, + stopAfterNoNew: 1, + postScanPrompt, + outputDir: join(f.directory, "output"), + }); + assert.equal(threadCount, 1); + assert.equal(prompts.length, 2); + assert.equal(prompts[1], postScanPrompt); + assert.equal(result.threadId, owner); + assert.equal(result.manifest.scan.status, "completed"); + assert.ok(result.manifest.scan.sealedAt); + assert.equal(result.manifest.scan.id, scanId); + assert.deepEqual(result.findings.findings, []); + assert.equal( + await readFile(result.manifestPath, "utf8"), + manifestBeforeFollowUp, + ); + assert.ok(result.cost === null || result.cost.inputTokens < 100_000); + assert.equal(result.toJSON().threadId, owner); + assert.ok( + (await readFile(join(f.directory, "output", "report.md"), "utf8")) + .length > 0, + ); + } finally { + await client.close(); + } + await assertSavedState(f, scanId, owner); + await assertExecutions(f, scanId, 4); +} + +async function assertSavedState(f, scanId, owner) { + const { deepScan } = await workbench(f, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + owner, + ]); + assert.equal(deepScan.status, "succeeded"); + if (deepScan.workflowVersion === "deep-security-scan/v2") { + const selected = deepScan.finalizationInput; + assert.ok( + selected, + "The completed v2 scan retains its finalization input.", + ); + assert.equal(selected.version, 1); + assert.equal(selected.terminalReason, deepScan.terminalReason); + assert.deepEqual(selected.omittedWorkerIds, []); + await assertDigest( + resolve(deepScan.scanDir, selected.resultPath), + selected.resultSha256, + ); + const workers = deepScan.workers.filter( + (worker) => worker.status === "succeeded", + ); + assert.equal(workers.length, 3); + for (const worker of workers) { + const attempt = deepScan.attempts.find( + (entry) => + entry.workerId === worker.id && entry.attempt === worker.attempt, + ); + assert.ok(attempt, "Each accepted worker retains its execution attempt."); + assert.equal(attempt.status, "succeeded"); + await assertDigest( + attempt.acceptedResultPath, + attempt.acceptedResultSha256, + ); + if (worker.kind === "dedup") { + assert.equal(selected.resultSha256, attempt.acceptedResultSha256); + } else { + const input = deepScan.dedupInputs.find( + (entry) => entry.discoveryWorkerId === worker.id, + ); + assert.ok(input, "The reducer retains each accepted discovery input."); + assert.equal(input.attempt, worker.attempt); + assert.equal(input.resultManifestSha256, attempt.acceptedResultSha256); + await assertDigest( + input.resultManifestPath, + input.resultManifestSha256, + ); + } + } + assert.equal(deepScan.dedupInputs.length, 2); + } + console.log( + JSON.stringify({ + fixture: basename(f.directory), + workflowVersion: deepScan.workflowVersion, + attempts: deepScan.attempts?.length ?? null, + selectedFinalization: deepScan.finalizationInput != null, + }), + ); +} + +async function assertDigest(path, expected) { + const bytes = await readFile(path); + assert.equal(createHash("sha256").update(bytes).digest("hex"), expected); +} + +async function assertDraft(path) { + const document = JSON.parse(await readFile(path, "utf8")); + const findings = JSON.parse( + await readFile(join(dirname(path), "findings.json"), "utf8"), + ); + assert.deepEqual(findings.findings, []); + assert.ok(document.scan.target); +} + +async function readExecutions(f) { + try { + return (await readFile(f.env.PACKAGE_DEEP_TRACE, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + } catch (error) { + if (error.code === "ENOENT") return []; + throw error; + } +} + +async function assertExecutions(f, scanId, preflights = 3) { + const executions = await readExecutions(f); + const workers = executions.filter((entry) => entry.phase === "worker"); + const reducers = executions.filter((entry) => entry.phase === "reducer"); + const incomplete = f.env.PACKAGE_DEEP_EMPTY_ONCE ? 1 : 0; + assert.equal(workers.length, 2 + incomplete); + assert.equal(workers.filter((entry) => !entry.complete).length, incomplete); + assert.equal(workers.filter((entry) => entry.resumed).length, incomplete); + assert.equal(reducers.length, 1); + assert.equal( + executions.filter((entry) => entry.phase === "preflight").length, + preflights, + ); + for (const execution of [...workers, ...reducers]) { + assert.equal(execution.scanId, scanId); + assert.equal(execution.home, f.home); + assert.equal(execution.hasApiKey, true); + assert.equal( + execution.args[execution.args.indexOf("--model") + 1], + "gpt-5.5", + ); + assert.ok(execution.args.includes('approval_policy="never"')); + } +} diff --git a/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs b/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs new file mode 100644 index 000000000..e59440558 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-deep-spawn.mjs @@ -0,0 +1,20 @@ +import childProcess from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +import { win32 } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Windows cannot execute the POSIX fixture's shebang. Preserve the selected +// native executable and its options, inserting only the deterministic protocol +// script, as in the worker launch tests. Every other child runs unchanged. +const executable = win32.toNamespacedPath(process.env.PACKAGE_DEEP_EXECUTABLE); +const script = fileURLToPath( + new URL("package-deep-codex.mjs", import.meta.url), +); +const spawn = childProcess.spawn; +childProcess.spawn = (command, args, options) => + spawn( + command, + win32.toNamespacedPath(command) === executable ? [script, ...args] : args, + options, + ); +syncBuiltinESMExports(); diff --git a/sdk/typescript/scripts/fixtures/package-rpc.mjs b/sdk/typescript/scripts/fixtures/package-rpc.mjs new file mode 100644 index 000000000..ceb6156f2 --- /dev/null +++ b/sdk/typescript/scripts/fixtures/package-rpc.mjs @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { createInterface } from "node:readline"; + +// This client deliberately uses only Node builtins. Detached plugin tests must +// not resolve an MCP client or SDK from the checkout's node_modules. +export async function startRpc(command, args, options) { + const { requestTimeoutMs = 30_000, ...spawnOptions } = options; + const child = spawn(command, args, { + ...spawnOptions, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + let sequence = 0; + let stderr = ""; + const pending = new Map(); + child.stderr.setEncoding("utf8").on("data", (chunk) => { + stderr += chunk; + }); + createInterface({ input: child.stdout }).on("line", (line) => { + const response = JSON.parse(line); + const waiter = pending.get(response.id); + if (!waiter) return; + pending.delete(response.id); + clearTimeout(waiter.timer); + if (response.error) + waiter.reject(new Error(JSON.stringify(response.error))); + else waiter.resolve(response.result); + }); + const exited = once(child, "exit"); + child.on("exit", (code, signal) => { + for (const waiter of pending.values()) { + clearTimeout(waiter.timer); + waiter.reject( + new Error(`Fixture RPC exited (${code}, ${signal}): ${stderr}`), + ); + } + pending.clear(); + }); + const client = { + child, + request(method, params = {}) { + return new Promise((resolve, reject) => { + const id = ++sequence; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`Fixture RPC timed out: ${method}\n${stderr}`)); + }, requestTimeoutMs); + pending.set(id, { resolve, reject, timer }); + child.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`, + ); + }); + }, + async call(name, args, meta) { + const result = await this.request("tools/call", { + name, + arguments: args, + ...(meta ? { _meta: meta } : {}), + }); + assert.notEqual(result.isError, true, JSON.stringify(result)); + return result.structuredContent ?? JSON.parse(result.content[0].text); + }, + async close() { + if (child.exitCode !== null || child.signalCode !== null) return; + child.stdin.end(); + const timeout = setTimeout(() => child.kill("SIGKILL"), 5_000); + try { + await exited; + } finally { + clearTimeout(timeout); + } + }, + }; + try { + await client.request("initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "installed-deep-fixture", version: "1.0.0" }, + }); + child.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" })}\n`, + ); + return client; + } catch (error) { + await client.close(); + throw error; + } +} diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs index 60945f75d..39f834a5d 100644 --- a/sdk/typescript/scripts/smoke-package.mjs +++ b/sdk/typescript/scripts/smoke-package.mjs @@ -808,6 +808,15 @@ try { ); await smokeNestedDeepScanWorker(installedRoot, consumer); + run( + process.execPath, + [ + join(packageRoot, "scripts", "fixtures", "package-deep-scan.mjs"), + installedRoot, + ], + { cwd: consumer }, + ); + console.log( `Validated installed ${packageManifest.name}@${packageManifest.version}: public import, NodeNext types, CLI, SDK lifecycle, credential locking, ${expectedPluginFiles.length} bundled plugin files, MCP initialization, bundled Codex version, dashboard assets, and a nested worker without global codex.`, ); diff --git a/sdk/typescript/src/accepted-audit.ts b/sdk/typescript/src/accepted-audit.ts new file mode 100644 index 000000000..ee526a955 --- /dev/null +++ b/sdk/typescript/src/accepted-audit.ts @@ -0,0 +1,49 @@ +export interface ScanDraftInput { + scanId: string; + complete?: boolean; + handoffClaimToken?: string; + scope?: Record; + threatModel?: Record; + findings: Record[]; + coverage: Record; +} + +/** Accepted evidence may still describe partial or unknown source coverage. */ +export interface AuditEvidence { + checkpoint?: ScanDraftInput; + accepted?: ScanDraftInput; +} + +export type AuditOutcome = AuditEvidence & + ( + | { status: "accepted"; execution: Execution; accepted: ScanDraftInput } + | { status: "checkpoint"; execution: Execution } + ); + +/** One attempt; enclosing callers own retries and public completion. */ +export async function runAcceptedAudit(input: { + signal: AbortSignal; + execute: () => Promise; + accept: (execution: Execution) => Promise; +}): Promise> { + input.signal.throwIfAborted(); + const execution = await input.execute(); + input.signal.throwIfAborted(); + const evidence = await input.accept(execution); + input.signal.throwIfAborted(); + return evidence.accepted === undefined + ? { ...evidence, execution, status: "checkpoint" } + : { + ...evidence, + execution, + status: "accepted", + accepted: evidence.accepted, + }; +} + +/** Process completion alone does not accept an unfinished audit checkpoint. */ +export function auditEvidence(checkpoint: ScanDraftInput): AuditEvidence { + return checkpoint.complete === false + ? { checkpoint } + : { checkpoint, accepted: checkpoint }; +} diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 425e486c4..ef6b318b2 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -1,5 +1,15 @@ /// +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { captureOriginalReasoningSummary } from "./reasoning-summary.js"; +export { scanPreflightCodexConfig } from "./preflight-config.js"; +import { resumeSelectedDeepScan } from "./deep-scan-finalization.js"; +import { + auditEvidence, + runAcceptedAudit, + type ScanDraftInput, +} from "./accepted-audit.js"; +import { pathToFileURL } from "node:url"; import { statSync } from "node:fs"; import { chmod, @@ -23,13 +33,15 @@ import { resolve, sep, } from "node:path"; -import { - Codex, - type CodexOptions, - type ThreadOptions, - type TurnOptions, -} from "@openai/codex-sdk"; +import { type CodexOptions, type ThreadOptions } from "@openai/codex-sdk"; import { z } from "incur"; +import { + createCodexClient, + readCodexSessionTurn, + type CodexSessionClient as CodexClientLike, + type CodexSessionThread as CodexThreadLike, + type CodexSessionEvent as ScanEvent, +} from "./codex-session.js"; import { CODEX_AUTH_CONFIG_KEYS, NO_CREDENTIALS_MESSAGE, @@ -68,6 +80,7 @@ import { type ScanCost, type ScanSessionEvent, } from "./cost.js"; +import type { ScanExecutionAttribution } from "./scan-sessions.js"; import { DeepScanProgressTracker, type DeepScanProgress, @@ -216,24 +229,6 @@ import { validateMode, } from "./targets.js"; -interface CodexThreadLike { - readonly id: string | null; - runStreamed( - input: string, - options: TurnOptions, - ): Promise<{ events: AsyncGenerator }>; -} - -interface ScanEvent { - readonly type: string; - readonly [key: string]: unknown; -} - -interface CodexClientLike { - startThread(options: ThreadOptions): CodexThreadLike; - resumeThread?(threadId: string, options: ThreadOptions): CodexThreadLike; -} - interface PreparedRuntime { codexHome: string; persistentCredentialHome?: boolean; @@ -453,7 +448,7 @@ interface ClientDependencies { } const DEFAULT_DEPENDENCIES: ClientDependencies = { - createCodex: (options) => new Codex(options), + createCodex: createCodexClient, environment: process.env, }; @@ -1010,15 +1005,7 @@ export class CodexSecurity { `permissions.${POLICY_PERMISSION_PROFILE}.filesystem=${inlineToml(policyFilesystemPermissions(inputs.gitMetadataPaths))}`, ], ); - const reportCost = (current: Readonly): void => { - const total = addScanCosts(accumulatedCost, current); - if (completeCost) - notifyObserver( - "onCost", - options.onCost, - options.onObserverError, - total, - ); + const enforceCostLimit = (total: Readonly): void => { if ( options.maxCostUsd !== undefined && total.estimatedUsd > options.maxCostUsd @@ -1030,6 +1017,17 @@ export class CodexSecurity { ); } }; + const reportCost = (current: Readonly): void => { + const total = addScanCosts(accumulatedCost, current); + if (completeCost) + notifyObserver( + "onCost", + options.onCost, + options.onObserverError, + total, + ); + enforceCostLimit(total); + }; const outputSchema = securityPolicyStageOutputSchema(); const run = async ( stage: SecurityPolicyStage, @@ -1053,6 +1051,10 @@ export class CodexSecurity { options.onCost === undefined && options.maxCostUsd === undefined ? undefined : reportCost, + onCostLowerBound: + options.maxCostUsd === undefined + ? undefined + : (cost) => enforceCostLimit(addScanCosts(accumulatedCost, cost)), onError: (error) => { if (options.maxCostUsd !== undefined) budgetController.abort(error); else @@ -1213,14 +1215,35 @@ export class CodexSecurity { let preparedTargetWarnings: string[] = []; let runPostScan: (() => ReturnType) | null = null; + let selectedDeepFinalization = false; + let observedScanThreadId: string | undefined; let activeScan: { id: string; options: WorkbenchCommandOptions; + mode: ScanMode; } | null = null; const prepareArtifactRestorer = this.#dependencies.prepareScanArtifactRestorer ?? prepareScanArtifactRestorer; const workbench = this.#dependencies.runWorkbench ?? runWorkbench; + const recoverCompletedScan = async ( + commandOptions: WorkbenchCommandOptions, + scanId: string, + error: unknown, + completionArgs: readonly string[], + ): Promise => { + const saved = await workbench(commandOptions, [ + "get-scan", + "--scan-id", + scanId, + ]).catch(() => null); + const savedScan = saved?.["scan"]; + const progress = isRecord(savedScan) ? savedScan["progress"] : null; + if (!isRecord(progress) || progress["status"] !== "complete") throw error; + // A lost response can follow a durable seal. Only the existing normal + // completion command can validate and return that committed receipt. + return workbench(commandOptions, completionArgs); + }; try { const checkOpen = (): void => { this.#requireOpen(); @@ -1282,13 +1305,12 @@ export class CodexSecurity { const { runtime, runtimeHome, - effectiveConfig, - preflightConfig, modelProvider, authentication, approvalPolicy, python, } = session; + let { effectiveConfig, preflightConfig } = session; releaseCredentialHome = session.releaseCredentialHome; const deepScanConfigPath = mode === "deep" @@ -1347,6 +1369,38 @@ export class CodexSecurity { ); checkOpen(); + if (mode === "deep" && options.resumeScanId === undefined) { + const summary = await captureOriginalReasoningSummary({ + config: session.sessionConfig, + command: this.#codexCommand(), + cwd: scanDir, + environment: { + ...withoutOpenAiApiKeys( + this.#createSessionEnvironment(session, {}, options.auth), + ), + ...(session.externalProvider === null && session.apiKey !== null + ? { CODEX_API_KEY: session.apiKey } + : {}), + }, + signal, + }); + if (summary !== undefined) { + effectiveConfig = { + ...effectiveConfig, + model_reasoning_summary: summary, + }; + preflightConfig = scanPreflightCodexConfig(effectiveConfig); + session.effectiveConfig = effectiveConfig; + session.preflightConfig = preflightConfig; + session.sessionConfig = { + ...session.sessionConfig, + model_reasoning_summary: summary, + }; + if (runtime.configPath !== undefined) + await writeCodexConfig(runtime.configPath, preflightConfig); + } + } + const shellPluginRoot = runtime.plugin.pluginRoot; const canonicalShellPluginRoot = await realpath(shellPluginRoot); const pluginRelativeToHome = relative( @@ -1437,6 +1491,14 @@ export class CodexSecurity { `Could not track scan activity: ${errorMessage(error)}`, ); }; + const enforceCostLimit = (cost: Readonly): boolean => { + if (maxCostUsd === undefined || cost.estimatedUsd <= maxCostUsd) + return false; + costAbortController.abort( + new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), + ); + return true; + }; const tracker = new ScanCostTracker({ codexHome: runtime.codexHome, model, @@ -1477,15 +1539,7 @@ export class CodexSecurity { cost, maxCostUsd, ); - if ( - maxCostUsd !== undefined && - cost.estimatedUsd > maxCostUsd - ) { - costAbortController.abort( - new ScanCostLimitExceededError(maxCostUsd, cost, scanDir), - ); - return; - } + if (enforceCostLimit(cost)) return; const request = options.onBudgetApproaching; if ( request === undefined || @@ -1550,6 +1604,8 @@ export class CodexSecurity { } }); }, + onCostLowerBound: + options.maxCostUsd === undefined ? undefined : enforceCostLimit, onError: reportTrackingError, }); costTracker = tracker; @@ -1722,7 +1778,23 @@ export class CodexSecurity { }, ); } - activeScan = { id: scanId, options: workbenchOptions }; + activeScan = { id: scanId, options: workbenchOptions, mode }; + if (mode === "deep") { + tracker.setAttributionReader(async () => { + const context = await workbench( + { ...workbenchOptions, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const scan = context["scan"]; + if (isRecord(scan) && !("executionAttribution" in scan)) + return undefined; + return isRecord(scan) && isRecord(scan["executionAttribution"]) + ? (scan[ + "executionAttribution" + ] as unknown as ScanExecutionAttribution) + : null; + }); + } if (mode === "deep" && options.onDeepProgress !== undefined) { let progressWarningReported = false; deepProgressTracker = new DeepScanProgressTracker({ @@ -1897,6 +1969,7 @@ export class CodexSecurity { ); } thread = codex.resumeThread(resumeThreadId, threadOptions); + observedScanThreadId = resumeThreadId; tracker.start(resumeThreadId); if (budgetRecovery !== null) budgetRecovery.threadId = resumeThreadId; await tracker.refresh().catch(reportTrackingError); @@ -1920,22 +1993,63 @@ export class CodexSecurity { if (postScanPrompt?.trim()) { runPostScan = () => thread.runStreamed(postScanPrompt, { signal }); } - const { events } = await thread.runStreamed(prompt, { - signal, - }); + const recoverSelectedCompletion = async () => { + const threadId = observedScanThreadId ?? thread.id; + if (mode !== "deep" || !threadId || signal.aborted) return null; + const saved = await workbench(workbenchOptions, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]).catch(() => null); + const deep = saved?.["deepScan"]; + if ( + !isRecord(deep) || + !isRecord(deep["finalizationInput"]) || + (deep["status"] !== "running" && deep["status"] !== "succeeded") + ) + return null; + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId, + threadId, + pluginRoot: runtime.plugin.installedRoot, + signal, + runWorkbench: (args) => workbench(workbenchOptions, args), + }); + return { + status: "completed" as const, + threadId, + finalResponse: "", + usage: null, + lastStreamError: null, + }; + }; + const savedCompletion = resumeThreadId + ? await recoverSelectedCompletion() + : null; + const events = (async function* () { + if (savedCompletion) return; + yield* (await thread.runStreamed(prompt, { signal })).events; + })(); checkOpen(); const result = await runScanEvents({ + savedCompletion: savedCompletion ?? undefined, + recoverCompletion: recoverSelectedCompletion, thread, events, signal, scanDir, + scanId, pluginRoot: runtime.plugin.installedRoot, expectation, authentication, workbenchValidated: true, model, onThreadStarted: async (threadId) => { + observedScanThreadId = threadId; if (resumeThreadId !== undefined) { if (threadId !== resumeThreadId) { throw new CodexSecurityError( @@ -1964,6 +2078,7 @@ export class CodexSecurity { } }, onFinalize: async (usage) => { + await recoverSelectedCompletion(); if (options.validationPrompt !== undefined) { tracker.recordUsage(usage); await tracker.refresh().catch(reportTrackingError); @@ -2029,7 +2144,10 @@ export class CodexSecurity { return { usage, cost: estimateScanCost(model, usage) }; }); throwIfAborted(signal, scanDir); - if (options.maxCostUsd !== undefined && snapshot.cost === null) { + if ( + options.maxCostUsd !== undefined && + (snapshot.cost === null || snapshot.cost.coverage === "partial") + ) { notifyObserver( "onWarning", options.onWarning, @@ -2096,14 +2214,20 @@ export class CodexSecurity { onObserverError: options.onObserverError, }); checkOpen(); - const completion = await workbench(workbenchOptions, [ + const completionArgs = [ "complete-scan", "--scan-id", scanId, ...(completionCost === null ? [] : ["--cost-json", JSON.stringify(completionCost)]), - ]); + ]; + const completion = await workbench( + workbenchOptions, + completionArgs, + ).catch((error) => + recoverCompletedScan(workbenchOptions, scanId, error, completionArgs), + ); activeScan = null; const completedScan = completion["scan"]; if (isRecord(completedScan) && Array.isArray(completedScan["warnings"])) { @@ -2151,7 +2275,7 @@ export class CodexSecurity { let artifactRestorer: ScanArtifactRestorer | null = null; try { artifactRestorer = await prepareArtifactRestorer( - workbenchOptions, + { ...workbenchOptions, signal: undefined }, scanDir, ); await runScanEvents({ @@ -2159,6 +2283,7 @@ export class CodexSecurity { events: (await followUp()).events, signal, scanDir, + scanId, pluginRoot: runtime.plugin.installedRoot, expectation, model, @@ -2168,7 +2293,6 @@ export class CodexSecurity { }); checkOpen(); } catch (error) { - if (signal.aborted || this.#closed) throw error; if (artifactRestorer !== null) { for (const artifact of completedArtifacts) { try { @@ -2185,6 +2309,7 @@ export class CodexSecurity { } } } + if (signal.aborted || this.#closed) throw error; await collectResult( result.turnResult, result.threadId, @@ -2278,17 +2403,57 @@ export class CodexSecurity { options.signal?.aborted !== true ) { try { - const completion = await workbench( - { ...activeScan.options, signal: undefined }, - [ - "complete-budget-exhausted-scan", + const budgetScanId = activeScan.id; + const budgetCost = snapshot?.cost ?? { lowerBound: failure.cost }; + const completionSignal = AbortSignal.any([ + this.#abortController.signal, + ...(options.signal === undefined ? [] : [options.signal]), + ]); + const completionOptions = { + ...activeScan.options, + signal: completionSignal, + }; + const saved = await workbench(completionOptions, [ + "get-deep-scan", + "--scan-id", + activeScan.id, + "--thread-id", + budgetRecovery.threadId, + ]).catch(() => null); + const deep = saved?.["deepScan"]; + if ( + isRecord(deep) && + deep["status"] === "running" && + isRecord(deep["finalizationInput"]) + ) { + // Cost stops model work, but an already selected result can still + // finish through the local publisher. Caller cancellation remains live. + selectedDeepFinalization = true; + await resumeSelectedDeepScan({ + scanId: activeScan.id, + threadId: budgetRecovery.threadId, + pluginRoot: budgetRecovery.pluginRoot, + signal: completionSignal, + runWorkbench: (args) => workbench(completionOptions, args), + }); + } + const completion = await workbench(completionOptions, [ + "complete-budget-exhausted-scan", + "--scan-id", + budgetScanId, + "--cost-json", + JSON.stringify(budgetCost), + "--message", + failure.message.slice(0, 2400), + ]).catch((error) => + recoverCompletedScan(completionOptions, budgetScanId, error, [ + "complete-scan", "--scan-id", - activeScan.id, - "--cost-json", - JSON.stringify(snapshot?.cost ?? failure.cost), - "--message", - failure.message.slice(0, 2400), - ], + budgetScanId, + ...(snapshot?.cost + ? ["--cost-json", JSON.stringify(snapshot.cost)] + : []), + ]), ); activeScan = null; runPostScan = null; @@ -2302,10 +2467,7 @@ export class CodexSecurity { scanDir, budgetRecovery.pluginRoot, budgetRecovery.expectation, - AbortSignal.any([ - this.#abortController.signal, - ...(options.signal === undefined ? [] : [options.signal]), - ]), + completionSignal, true, ); if (result.coverage.completeness !== "partial") { @@ -2344,9 +2506,32 @@ export class CodexSecurity { return result; } catch {} } - // A failed attachment must not turn a resumable coordinator into a terminal failure. - // Deep Scan orchestration persists its own terminal failures and cancellations. - if (activeScan !== null && options.resumeScanId === undefined) { + const callerCanceledDeepScan = + activeScan?.mode === "deep" && options.signal?.aborted; + if (activeScan !== null && callerCanceledDeepScan) { + const workbenchOptions = { ...activeScan.options, signal: undefined }; + // The workbench owns the running-state check and repeated cancellation. + // Selection may have committed before the SDK received its response. + await workbench(workbenchOptions, [ + "cancel-scan", + "--scan-id", + activeScan.id, + ...(observedScanThreadId === undefined + ? [] + : ["--thread-id", observedScanThreadId]), + ]).catch(() => undefined); + } + // Publication failures remain resumable. A cost stop or explicit client close + // still uses the existing failure path to retain partial results and stop work. + if ( + activeScan !== null && + !callerCanceledDeepScan && + ((options.resumeScanId === undefined && !selectedDeepFinalization) || + (selectedDeepFinalization && + !options.signal?.aborted && + (failure instanceof ScanCostLimitExceededError || + this.#abortController.signal.aborted))) + ) { if ( options.validationPrompt !== undefined && !customValidationComplete @@ -2619,13 +2804,11 @@ export class CodexSecurity { } } - #createSessionCodex( + #createSessionEnvironment( session: PreparedSession, runtimePaths: Record, auth: ScanAuthMode = "auto", - config?: JsonObject, - configOverrides: string[] = [], - ): { codex: CodexClientLike; environment: ProcessEnvironment } { + ): ProcessEnvironment { const { runtime, python, @@ -2661,6 +2844,23 @@ export class CodexSecurity { if (session.safetyIdentifier !== undefined) { environment[SAFETY_IDENTIFIER_ENV] = session.safetyIdentifier; } + return environment; + } + + #createSessionCodex( + session: PreparedSession, + runtimePaths: Record, + auth: ScanAuthMode = "auto", + config?: JsonObject, + configOverrides: string[] = [], + ): { codex: CodexClientLike; environment: ProcessEnvironment } { + const { externalProvider, apiKey, sessionConfig } = session; + const commandAuth = hasCommandAuth(sessionConfig); + const environment = this.#createSessionEnvironment( + session, + runtimePaths, + auth, + ); const sdkCodexConfig = { ...(config ?? sessionConfig) }; // Projects and permissions already live in generated TOML files; the SDK // cannot safely encode their path and selector keys as dotted overrides. @@ -3612,6 +3812,11 @@ async function removeTargetPathsFile(path: string | null): Promise { } interface ScanEventRunOptions { + scanId?: string; + savedCompletion?: Awaited>; + recoverCompletion?: () => Promise + > | null>; thread: CodexThreadLike; events: AsyncGenerator; signal: AbortSignal; @@ -3645,112 +3850,167 @@ export async function runScanEvents( let scanStarted = false; let tacStatusReported = false; try { - const turn = await readCodexTurn({ - thread: options.thread, - events: options.events, - onEvent: async (event) => { - if (!tacStatusReported) { - const tacStatus = trustedAccessStatusFromEvent(event); - if (tacStatus !== null) { - tacStatusReported = true; - notifyObserver( - "onTrustedAccessStatus", - options.onTrustedAccessStatus, - options.onObserverError, - tacStatus, - ); - if (tacStatus !== "granted") { + let completedTurn: + | (Awaited> & { + threadId: string; + status: "completed"; + }) + | undefined; + const execute = async () => { + const turn = + options.savedCompletion ?? + (await readCodexTurn({ + thread: options.thread, + events: options.events, + onEvent: async (event) => { + if (!tacStatusReported) { + const tacStatus = trustedAccessStatusFromEvent(event); + if (tacStatus !== null) { + tacStatusReported = true; + notifyObserver( + "onTrustedAccessStatus", + options.onTrustedAccessStatus, + options.onObserverError, + tacStatus, + ); + if (tacStatus !== "granted") { + notifyObserver( + "onWarning", + options.onWarning, + options.onObserverError, + trustedAccessWarning(tacStatus, options.authentication), + ); + } + } + } + for (const activity of scanActivitiesFromEvent( + event, + options.expectation.repository, + )) { notifyObserver( - "onWarning", - options.onWarning, + "onActivity", + options.onActivity, options.onObserverError, - trustedAccessWarning(tacStatus, options.authentication), + activity, ); } - } - } - for (const activity of scanActivitiesFromEvent( - event, - options.expectation.repository, - )) { - notifyObserver( - "onActivity", - options.onActivity, - options.onObserverError, - activity, - ); - } - for (const progress of scanProgressUpdatesFromEvent(event)) { - if ( - options.expectedFilesTotal !== undefined && - progress.filesTotal !== options.expectedFilesTotal - ) { - continue; - } - notifyObserver( - "onProgress", - options.onProgress, - options.onObserverError, - progress, - ); - } - const workerStatus = workerStatusFromEvent(event); - if (workerStatus !== null) { - notifyObserver( - "onWorkerStatus", - options.onWorkerStatus, - options.onObserverError, - workerStatus, - ); - } - if (event.type === "thread.started") { - const startedThreadId = event["thread_id"]; - if (typeof startedThreadId === "string") { - await options.onThreadStarted?.(startedThreadId); - } - if (!scanStarted) { - scanStarted = true; + for (const progress of scanProgressUpdatesFromEvent(event)) { + if ( + options.expectedFilesTotal !== undefined && + progress.filesTotal !== options.expectedFilesTotal + ) { + continue; + } + notifyObserver( + "onProgress", + options.onProgress, + options.onObserverError, + progress, + ); + } + const workerStatus = workerStatusFromEvent(event); + if (workerStatus !== null) { + notifyObserver( + "onWorkerStatus", + options.onWorkerStatus, + options.onObserverError, + workerStatus, + ); + } + if (event.type === "thread.started") { + const startedThreadId = event["thread_id"]; + if (typeof startedThreadId === "string") { + await options.onThreadStarted?.(startedThreadId); + } + if (!scanStarted) { + scanStarted = true; + notifyObserver( + "onScanStarted", + options.onScanStarted, + options.onObserverError, + ); + } + } + }, + onReconnect: (message, reconnect) => { notifyObserver( - "onScanStarted", - options.onScanStarted, + "onReconnect", + options.onReconnect, options.onObserverError, + ...reconnect, + reconnectDetails(message), ); - } - } - }, - onReconnect: (message, reconnect) => { - notifyObserver( - "onReconnect", - options.onReconnect, - options.onObserverError, - ...reconnect, - reconnectDetails(message), + }, + })); + const { status, threadId, lastStreamError } = turn; + if (status !== "completed") { + throw new IncompleteScanError( + lastStreamError ?? + "Codex Security event stream ended before the turn completed.", ); - }, - }); - const { status, threadId, finalResponse, lastStreamError } = turn; - let { usage } = turn; - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - if (status !== "completed") { - throw new IncompleteScanError( - lastStreamError ?? - "Codex Security event stream ended before the turn completed.", + } + if (threadId === null) { + throw new IncompleteScanError( + "Codex Security did not report a thread ID.", + ); + } + return (completedTurn = { ...turn, threadId, status }); + }; + const accept = async () => { + // Matching, custom validation and the canonical seal remain with the caller. + const [manifest, findings, coverage] = await Promise.all( + ["scan-manifest.json", "findings.json", "coverage.json"].map( + async (name) => + JSON.parse( + ( + await readScanFile(options.scanDir, name, name, options.signal) + ).toString("utf8"), + ), + ), ); + const helper = ( + await import( + pathToFileURL(join(options.pluginRoot, "mcp/helpers.mjs")).href + ) + ).default; + const draft: ScanDraftInput = helper.parseCanonicalScanDraft({ + scanId: options.scanId ?? manifest.scan.id, + manifest, + findings, + coverage, + }); + return auditEvidence(draft); + }; + let audit; + try { + audit = await runAcceptedAudit({ + signal: options.signal, + execute, + accept, + }); + } catch (error) { + if (options.signal.aborted) throw error; + const saved = await options.recoverCompletion?.(); + if (!saved || saved.status !== "completed" || saved.threadId === null) + throw error; + // The enclosing scan publishes its saved selection; acceptance then reads it. + const recovered = completedTurn ?? { ...saved, threadId: saved.threadId }; + audit = await runAcceptedAudit({ + signal: options.signal, + execute: async () => recovered, + accept, + }); } - if (threadId === null) { + if (audit.status === "checkpoint") throw new IncompleteScanError( - "Codex Security did not report a thread ID.", + "Codex Security produced only an unfinished audit checkpoint.", ); - } + const { status, threadId, finalResponse } = audit.execution; + let { usage } = audit.execution; if (options.onFinalize !== undefined) { usage = (await options.onFinalize(usage)) ?? usage; } - const result = await collectResult( + return await collectResult( { status, finalResponse, @@ -3764,13 +4024,6 @@ export async function runScanEvents( options.signal, options.workbenchValidated, ); - if (options.signal.aborted) { - throw new ScanInterruptedError( - `Codex Security scan was interrupted; partial output remains at ${options.scanDir}.`, - options.scanDir, - ); - } - return result; } catch (error) { if (options.signal.reason instanceof ScanCostLimitExceededError) { throw options.signal.reason; @@ -3798,61 +4051,28 @@ async function readCodexTurn(options: { usage: unknown; lastStreamError: string | null; }> { - let threadId = options.thread.id; - let status: "in_progress" | "completed" = "in_progress"; - let finalResponse = ""; - let usage: unknown = null; - let lastStreamError: string | null = null; - for await (const event of eventsWithOptionalUsage(options.events)) { - await options.onEvent?.(event); - if ( - event.type === "thread.started" && - typeof event["thread_id"] === "string" - ) { - threadId = event["thread_id"]; - } else if ( - event.type === "item.completed" && - isRecord(event["item"]) && - event["item"]["type"] === "agent_message" && - typeof event["item"]["text"] === "string" - ) { - finalResponse = event["item"]["text"]; - } else if (event.type === "turn.completed") { - status = "completed"; - usage = event["usage"]; - } else if (event.type === "turn.failed") { - throw new CodexSecurityError(turnFailureMessage(event["error"])); - } else if (event.type === "error" && typeof event["message"] === "string") { - const message = event["message"]; - const classification = classifyConnectionFailure(message); - if (classification === "unauthorized" || classification === "forbidden") { - throw new CodexSecurityError(message); + return readCodexSessionTurn({ + ...options, + onEvent: async (event) => { + await options.onEvent?.(event); + if (event.type === "turn.failed") { + throw new CodexSecurityError(turnFailureMessage(event["error"])); } - const reconnect = reconnectAttempt(message); - if (reconnect === null) throw new CodexSecurityError(message); - lastStreamError = message; - options.onReconnect?.(message, reconnect); - } - } - return { threadId, status, finalResponse, usage, lastStreamError }; -} - -async function* eventsWithOptionalUsage( - events: AsyncGenerator, -): AsyncGenerator { - try { - yield* events; - } catch (error) { - if ( - error instanceof TypeError && - /\b(?:null|undefined)\b/u.test(error.message) && - /\bcache_write_input_tokens\b/u.test(error.message) - ) { - yield { type: "turn.completed", usage: null }; - return; - } - throw error; - } + if (event.type === "error" && typeof event["message"] === "string") { + const message = event["message"]; + const classification = classifyConnectionFailure(message); + if ( + classification === "unauthorized" || + classification === "forbidden" + ) { + throw new CodexSecurityError(message); + } + const reconnect = reconnectAttempt(message); + if (reconnect === null) throw new CodexSecurityError(message); + options.onReconnect?.(message, reconnect); + } + }, + }); } function trustedAccessStatusFromEvent( @@ -3958,7 +4178,7 @@ function scanPrompt( "Run this Codex Security scan non-interactively.", ...(mode === "deep" ? [ - `The SDK has already registered this scan. Call start_codex_security_deep_scan with ${JSON.stringify({ scanId })}; never pass targetPath or create another scan.`, + `The SDK has already registered this scan. Call start_codex_security_deep_scan with ${JSON.stringify({ scanId })}; never pass targetPath or create another scan. Leave finalization to the SDK after this turn; do not call a completion tool.`, ] : skillName === "security-scan" || customValidation ? [ @@ -4018,7 +4238,7 @@ function scanPrompt( ] : skillName === "deep-security-scan" ? [ - "The Deep Scan coordinator already wrote the canonical scan artifacts. Call complete_codex_security_scan exactly once without submitting another semantic draft; the workbench owns authoritative metadata, finalization, report generation, and sealing.", + "The Deep Scan coordinator already wrote the canonical scan artifacts. Leave them unchanged and end the scan turn without calling a completion tool; the SDK accounts for the completed turn, enforces its budget, and owns finalization, report generation, and sealing.", ] : [ "Use record_codex_security_scan_draft and complete_codex_security_scan as directed by the selected skill; the workbench owns authoritative metadata, finalization, report generation, and sealing.", @@ -4140,6 +4360,23 @@ function addScanCosts( previous.cacheWriteInputTokens + current.cacheWriteInputTokens, outputTokens: previous.outputTokens + current.outputTokens, estimatedUsd: previous.estimatedUsd + current.estimatedUsd, + ...(previous.coverage === "partial" || current.coverage === "partial" + ? { coverage: "partial" as const } + : {}), + ...(previous.modelCosts || + current.modelCosts || + previous.model !== current.model + ? { + modelCosts: [ + ...(previous.modelCosts ?? [previous]), + ...(current.modelCosts ?? [current]), + ], + } + : {}), + ...(previous.cacheWriteInputTokensReported === false || + current.cacheWriteInputTokensReported === false + ? { cacheWriteInputTokensReported: false } + : {}), }; } @@ -4605,136 +4842,6 @@ function sharedCredentialCodexConfig( return scanRuntimeCodexConfig(shared, credentialHome); } -export function scanPreflightCodexConfig(config: JsonObject): JsonObject { - const safeString = (value: unknown): value is string => - typeof value === "string" && - value.length > 0 && - !/[\u0000-\u001f\u007f]/u.test(value); - const safeProfileName = (value: unknown): value is string => - safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); - const safeInteger = (value: unknown): value is number => - typeof value === "number" && Number.isSafeInteger(value) && value >= 0; - const capabilityFeatures = (value: unknown): JsonObject => { - if (!isRecord(value)) return {}; - const result: JsonObject = {}; - for (const key of ["goals", "multi_agent", "enable_fanout"]) { - if (typeof value[key] === "boolean") result[key] = value[key]; - } - const multiAgent = value["multi_agent_v2"]; - if (typeof multiAgent === "boolean") { - result["multi_agent_v2"] = multiAgent; - } else if (isRecord(multiAgent)) { - const sanitized: JsonObject = {}; - if (typeof multiAgent["enabled"] === "boolean") { - sanitized["enabled"] = multiAgent["enabled"]; - } - const capacity = multiAgent["max_concurrent_threads_per_session"]; - if (safeInteger(capacity)) { - sanitized["max_concurrent_threads_per_session"] = capacity; - } - if (Object.keys(sanitized).length > 0) { - result["multi_agent_v2"] = sanitized; - } - } - return result; - }; - const executionConfig = (source: JsonObject): JsonObject => { - const result: JsonObject = {}; - for (const key of [ - "model", - "model_reasoning_effort", - "model_reasoning_summary", - "model_provider", - "service_tier", - ]) { - const value = source[key]; - if (safeString(value)) result[key] = value; - } - const features = capabilityFeatures(source["features"]); - if (Object.keys(features).length > 0) result["features"] = features; - const agents = source["agents"]; - if (isRecord(agents)) { - const sanitized: JsonObject = {}; - for (const key of ["max_threads", "max_depth"]) { - const value = agents[key]; - if (safeInteger(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; - } - const multiagent = source["multiagent_config"]; - if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { - result["multiagent_config"] = { - max_concurrency: multiagent["max_concurrency"], - }; - } - return result; - }; - const result = executionConfig(config); - // Keep the effective summary even when preflight filters the profile name. - const reasoningSummary = - resolveCodexProfile(config)["model_reasoning_summary"]; - if (safeString(reasoningSummary)) { - result["model_reasoning_summary"] = reasoningSummary; - } - const selectedProfile = safeProfileName(config["profile"]) - ? config["profile"] - : undefined; - if (selectedProfile !== undefined) { - result["profile"] = selectedProfile; - } - const profiles = config["profiles"]; - if (isRecord(profiles)) { - const sanitized: JsonObject = {}; - for (const [name, profile] of Object.entries(profiles)) { - if (!safeProfileName(name) || !isRecord(profile)) continue; - const projected = executionConfig(profile as JsonObject); - if (Object.keys(projected).length === 0) continue; - sanitized[name] = projected; - } - if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; - } - const modelProvider = scanModelProvider(result); - if (isExternalModelProvider(modelProvider)) { - result["model_providers"] = { - [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, - }; - } else if (modelProvider === "amazon-bedrock") { - const providers = config["model_providers"]; - const provider = isRecord(providers) ? providers[modelProvider] : undefined; - const aws = isRecord(provider) ? provider["aws"] : undefined; - if (isRecord(aws)) { - const sanitized: JsonObject = {}; - for (const key of ["region", "profile"]) { - const value = aws[key]; - if (safeString(value)) sanitized[key] = value; - } - if (Object.keys(sanitized).length > 0) { - result["model_providers"] = { - [modelProvider]: { aws: sanitized }, - }; - } - } - } - const rootMarkers = config["project_root_markers"]; - if (Array.isArray(rootMarkers)) { - result["project_root_markers"] = rootMarkers.filter(safeString); - } - const projects = config["projects"]; - if (isRecord(projects)) { - const sanitized: JsonObject = {}; - for (const [path, project] of Object.entries(projects)) { - if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { - continue; - } - const trust = project["trust_level"]; - if (trust !== "trusted" && trust !== "untrusted") continue; - sanitized[path] = { trust_level: trust }; - } - if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; - } - return result; -} - async function pluginSupportsIsolatedDeepScanConfig( pluginRoot: string, ): Promise { diff --git a/sdk/typescript/src/cli-scan-logs-json.ts b/sdk/typescript/src/cli-scan-logs-json.ts new file mode 100644 index 000000000..8bceaacc5 --- /dev/null +++ b/sdk/typescript/src/cli-scan-logs-json.ts @@ -0,0 +1,31 @@ +import { Formatter } from "incur"; +import type { readSavedScanLogs } from "./scan-logs.js"; + +// Format records separately: a saved log document can exceed the runtime's +// maximum string length even though each session and event fits comfortably. +export async function* scanLogsJson( + logs: Awaited>, + cta?: unknown, +): AsyncGenerator { + yield Buffer.from( + `{\n "scanId": ${JSON.stringify(logs.scanId)},\n "threadId": ${JSON.stringify(logs.threadId)}`, + ); + for (const key of ["sessions", "events"] as const) { + yield Buffer.from(`,\n "${key}": [`); + let first = true; + for (const record of logs[key]) { + const formatted = Formatter.format(record, "json").replaceAll( + "\n", + "\n ", + ); + yield Buffer.from(`${first ? "\n" : ",\n"} ${formatted}`); + first = false; + } + yield Buffer.from(first ? "]" : "\n ]"); + } + if (cta !== undefined) { + const formatted = Formatter.format(cta, "json").replaceAll("\n", "\n "); + yield Buffer.from(`,\n "cta": ${formatted}`); + } + yield Buffer.from("\n}\n"); +} diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 9c00f32af..a44de3ae6 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -45,6 +45,7 @@ import { pipeline } from "node:stream/promises"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify, stripVTControlCharacters } from "node:util"; import { Cli, z } from "incur"; +import { scanLogsJson } from "./cli-scan-logs-json.js"; import { parse as parseToml } from "smol-toml"; import { classifyConnectionFailure, @@ -1773,6 +1774,7 @@ export async function main( let exitCode = 0; let frameworkExit: number | undefined; let frameworkOutput = ""; + let streamedLogs: Awaited> | undefined; let renderedHistory: string | undefined; let renderedPublication: string | undefined; let renderedPolicy: string | undefined; @@ -2120,18 +2122,33 @@ export async function main( .describe("Scan identifier or unique prefix (default: latest)."), }), output: z.record(z.string(), z.unknown()).optional(), - async run({ args }) { + async run({ args, format }) { const scanId = args.scanId ?? (await latestScans(1, "any"))?.[0]?.scanId; if (scanId === undefined) return; - return await history( + const result = await history( ["get-scan", "--scan-id", scanId], - async (value) => - (await readSavedScanLogs( + async (value) => { + const logs = await readSavedScanLogs( value["scan"] as ScanLogSource, codexSecurityCredentialHome(dependencies.environment), - )) as unknown as JsonObject, + ); + // Incur owns filtering, envelopes and token controls. Keep those + // requests on its formatter; plain JSON needs no aggregate string. + if ( + format === "json" && + !argv.some((argument) => + /^--(?:filter-output|full-output|token-count|token-limit|token-offset)(?:=|$)/u.test( + argument, + ), + ) + ) { + streamedLogs = logs; + } + return logs as unknown as JsonObject; + }, ); + return streamedLogs === undefined ? result : undefined; }, }) .command("resume", { @@ -5682,11 +5699,21 @@ export async function main( return 2; } } - if (frameworkOutput.length === 0) return exitCode; + if (frameworkOutput.length === 0 && streamedLogs === undefined) + return exitCode; try { + // Incur can add a stale-skills CTA after the logs handler returns. + const logOutput = + streamedLogs === undefined + ? undefined + : scanLogsJson( + streamedLogs, + frameworkOutput ? JSON.parse(frameworkOutput).cta : undefined, + ); await writeCliOutput( output, - renderedPolicy ?? + logOutput ?? + renderedPolicy ?? renderedPatch ?? renderedPublication ?? renderedHistory ?? diff --git a/sdk/typescript/src/codex-session.ts b/sdk/typescript/src/codex-session.ts new file mode 100644 index 000000000..630df3625 --- /dev/null +++ b/sdk/typescript/src/codex-session.ts @@ -0,0 +1,94 @@ +import { + Codex, + type CodexOptions, + type ThreadOptions, + type TurnOptions, +} from "@openai/codex-sdk"; + +export interface CodexSessionEvent { + readonly type: string; + readonly [key: string]: unknown; +} + +export interface CodexSessionThread { + readonly id: string | null; + runStreamed( + input: string, + options: TurnOptions, + ): Promise<{ events: AsyncGenerator }>; +} + +export interface CodexSessionClient { + startThread(options: ThreadOptions): CodexSessionThread; + resumeThread?(threadId: string, options: ThreadOptions): CodexSessionThread; +} + +export const createCodexClient = (options: CodexOptions): CodexSessionClient => + new Codex(options); + +/** Reduce a single stream; callers retain error, retry and acceptance policy. */ +export async function readCodexSessionTurn(options: { + thread: CodexSessionThread; + events: AsyncGenerator; + onEvent: (event: CodexSessionEvent) => Promise | void; + stopOnCompletion?: boolean; +}): Promise<{ + threadId: string | null; + status: "in_progress" | "completed"; + finalResponse: string; + usage: unknown; + lastStreamError: string | null; +}> { + let threadId = options.thread.id; + let status: "in_progress" | "completed" = "in_progress"; + let finalResponse = ""; + let usage: unknown = null; + let lastStreamError: string | null = null; + for await (const event of eventsWithOptionalUsage(options.events)) { + await options.onEvent(event); + if ( + event.type === "thread.started" && + typeof event["thread_id"] === "string" + ) { + threadId = event["thread_id"]; + } else if ( + event.type === "item.completed" && + isRecord(event["item"]) && + event["item"]["type"] === "agent_message" && + typeof event["item"]["text"] === "string" + ) { + finalResponse = event["item"]["text"]; + } else if (event.type === "turn.completed") { + status = "completed"; + usage = event["usage"] ?? null; + if (options.stopOnCompletion) break; + } else if (event.type === "error" && typeof event["message"] === "string") { + lastStreamError = event["message"]; + } + } + return { threadId, status, finalResponse, usage, lastStreamError }; +} + +async function* eventsWithOptionalUsage( + events: AsyncGenerator, +): AsyncGenerator { + try { + yield* events; + } catch (error) { + // The pinned SDK accesses this field before yielding a completion with + // absent usage. Preserve completion without inventing a zero-token receipt. + if ( + error instanceof TypeError && + /\b(?:null|undefined)\b/u.test(error.message) && + /\bcache_write_input_tokens\b/u.test(error.message) + ) { + yield { type: "turn.completed", usage: null }; + return; + } + throw error; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/sdk/typescript/src/cost-model.ts b/sdk/typescript/src/cost-model.ts index f24dcd4a2..c6781bfb5 100644 --- a/sdk/typescript/src/cost-model.ts +++ b/sdk/typescript/src/cost-model.ts @@ -6,6 +6,8 @@ export interface ScanCost { cacheWriteInputTokensReported?: boolean; outputTokens: number; estimatedUsd: number; + coverage?: "partial"; + modelCosts?: readonly ScanCost[]; pricing?: { source: string; asOf: string; @@ -96,6 +98,102 @@ export function tokenUsage(value: unknown): ScanTokenUsage | null { export function estimateScanCost( model: string | undefined, usage: unknown, +): ScanCost | null { + if (isRecord(usage) && Array.isArray(usage["modelUsage"])) { + const total = tokenUsage(usage); + if (total === null || usage["modelUsage"].length === 0) return null; + const costs: ScanCost[] = []; + for (const part of usage["modelUsage"]) { + if (!isRecord(part) || typeof part["model"] !== "string") return null; + const cost = estimateModelCost(part["model"], part); + if (cost === null) return null; + costs.push(cost); + } + const sum = ( + key: + | "inputTokens" + | "cachedInputTokens" + | "cacheWriteInputTokens" + | "outputTokens" + | "estimatedUsd", + ) => costs.reduce((value, cost) => value + cost[key], 0); + if ( + sum("inputTokens") !== total.input_tokens || + sum("cachedInputTokens") !== total.cached_input_tokens || + sum("cacheWriteInputTokens") !== total.cache_write_input_tokens || + sum("outputTokens") !== total.output_tokens + ) + return null; + return { + model: model ?? costs[0]!.model, + inputTokens: total.input_tokens, + cachedInputTokens: total.cached_input_tokens, + cacheWriteInputTokens: total.cache_write_input_tokens, + ...(total.cache_write_input_tokens_reported === false + ? { cacheWriteInputTokensReported: false } + : {}), + outputTokens: total.output_tokens, + estimatedUsd: sum("estimatedUsd"), + modelCosts: costs, + ...(costs.length === 1 ? { pricing: costs[0]!.pricing } : {}), + ...(usage["coverage"] === "partial" + ? { coverage: "partial" as const } + : {}), + }; + } + const cost = estimateModelCost(model, usage); + return cost && isRecord(usage) && usage["coverage"] === "partial" + ? { ...cost, coverage: "partial" } + : cost; +} + +// Internal budget enforcement only. The public estimate remains unavailable +// when some attributed usage has no price. +export function estimateScanCostLowerBound( + model: string | undefined, + usage: unknown, +): ScanCost | null { + if (!isRecord(usage) || !Array.isArray(usage["modelUsage"])) return null; + const total = tokenUsage(usage); + if (total === null) return null; + const keys = [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_output_tokens", + ] as const; + const observed = Object.fromEntries(keys.map((key) => [key, 0])); + const priced = Object.fromEntries(keys.map((key) => [key, 0])); + const parts: Record[] = []; + let cacheWritesReported = true; + for (const part of usage["modelUsage"]) { + const normalized = tokenUsage(part); + if (!isRecord(part) || normalized === null) return null; + for (const key of keys) observed[key]! += normalized[key]; + if ( + typeof part["model"] !== "string" || + estimateModelCost(part["model"], part) === null + ) + continue; + parts.push(part); + for (const key of keys) priced[key]! += normalized[key]; + if (normalized.cache_write_input_tokens_reported === false) + cacheWritesReported = false; + } + // A malformed partition is not evidence of an enforceable lower bound. + if (keys.some((key) => observed[key] !== total[key])) return null; + return estimateScanCost(model, { + ...priced, + cache_write_input_tokens_reported: cacheWritesReported, + modelUsage: parts, + coverage: "partial", + }); +} + +function estimateModelCost( + model: string | undefined, + usage: unknown, ): ScanCost | null { if (model === undefined) return null; const pricingModel = model.startsWith("openai.") diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index 2263f136f..c5042fdc2 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -1,7 +1,9 @@ -import { open, readdir } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { open, readdir, realpath } from "node:fs/promises"; import { join } from "node:path"; import { estimateScanCost, + estimateScanCostLowerBound, tokenUsage, type ScanCost, type ScanTokenUsage, @@ -11,9 +13,13 @@ import { type ScanActivity, } from "./scan-activity.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, + recordedScanCodexHome, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; import { scanProgressUpdatesFromEvent, @@ -48,6 +54,16 @@ interface SessionUsage { inheritedUsage: ScanTokenUsage | null; replaying: boolean; usage: ScanTokenUsage | null; + counterUsage: ScanTokenUsage | null; + model: string | null; + modelUsage: Map; + currentTurnId: string | null; + previousUsage: ScanTokenUsage | null; + responseIds: Set; + responseUsageObserved: boolean; + responseTokens: number; + expectedResponseTokens: number; + counterRegressed: boolean; calls: Map; activities: ScanActivity[]; progress: ScanProgress[]; @@ -56,7 +72,8 @@ interface SessionUsage { prose: Set; reasoning: SessionReasoning | null; reasoningCount: number; - events?: Record[]; + eventIndex: number; + events?: { index: number; event: Record }[]; } interface ScanCostTrackerOptions { @@ -67,6 +84,8 @@ interface ScanCostTrackerOptions { maxCostUsd?: number; expectedFilesTotal?: number; onCost?: (cost: Readonly) => void; + // Only reported when the full public estimate is unavailable. + onCostLowerBound?: (cost: Readonly) => void; onActivity?: (activity: ScanActivity) => void; onProgress?: (progress: ScanProgress) => void; onSessionEvent?: (event: ScanSessionEvent) => void; @@ -94,6 +113,16 @@ function createSessionUsage(): SessionUsage { inheritedUsage: null, replaying: false, usage: null, + counterUsage: null, + model: null, + modelUsage: new Map(), + currentTurnId: null, + previousUsage: null, + responseIds: new Set(), + responseUsageObserved: false, + responseTokens: 0, + expectedResponseTokens: 0, + counterRegressed: false, calls: new Map(), activities: [], progress: [], @@ -102,6 +131,7 @@ function createSessionUsage(): SessionUsage { prose: new Set(), reasoning: null, reasoningCount: 0, + eventIndex: 0, }; } @@ -112,13 +142,18 @@ export class ScanCostTracker { readonly #workers = new Map(); readonly #workerProgress = new Map(); readonly #reportedProgress = new Set(); + readonly #reportedSessionEvents = new Map>(); #threadId: string | null = null; #timer: NodeJS.Timeout | null = null; #pending: Promise = Promise.resolve(); #snapshot: ScanCostSnapshot = { usage: null, cost: null }; #lastCost: string | null = null; + #lastCostLowerBound: string | null = null; #highestFilesCompleted = 0; #expectedFilesTotal: number | undefined; + #attribution: ScanExecutionAttribution | null = null; + #readAttribution: + (() => Promise) | undefined; public constructor(options: ScanCostTrackerOptions) { this.#options = options; @@ -129,10 +164,23 @@ export class ScanCostTracker { this.#expectedFilesTotal = filesTotal; } + public setAttributionReader( + reader: () => Promise, + ): void { + this.#readAttribution = reader; + } + public recordUsage(usage: unknown, threadId = this.#threadId): void { const normalized = tokenUsage(usage); if (threadId !== null) { - this.#receipts.set(threadId, normalized); + const previous = this.#receipts.get(threadId); + if ( + previous == null || + (normalized !== null && + normalized.total_tokens >= previous.total_tokens) + ) { + this.#receipts.set(threadId, normalized); + } } } @@ -142,6 +190,7 @@ export class ScanCostTracker { if ( this.#options.maxCostUsd === undefined && this.#options.onCost === undefined && + this.#options.onCostLowerBound === undefined && this.#options.onActivity === undefined && this.#options.onProgress === undefined && this.#options.onSessionEvent === undefined @@ -189,35 +238,76 @@ export class ScanCostTracker { } if (fallbackUsage !== undefined) this.recordUsage(fallbackUsage); await this.refresh(); - if (this.#receipts.size > 0 || this.#snapshot.usage !== null) + if ( + this.#readAttribution || + this.#receipts.size > 0 || + this.#snapshot.usage !== null + ) return this.#snapshot; const cost = estimateScanCost(this.#options.model, fallbackUsage); this.#snapshot = { usage: fallbackUsage ?? null, cost }; - this.#reportCost(cost); + this.#reportCost(cost, fallbackUsage); return this.#snapshot; } async #readSessions(): Promise { if (this.#threadId === null) return; - const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; - for await (const path of sessionFiles( - join(this.#options.codexHome, "sessions"), - )) { - let session = this.#sessions.get(path); - if (session === undefined) { - session = createSessionUsage(); - this.#sessions.set(path, session); + if (this.#readAttribution) { + const record = await this.#readAttribution(); + if (record === null) return; + const attribution = record === undefined || record.legacy ? null : record; + if ( + attribution && + (!this.#attribution || + attribution.completedAt !== this.#attribution.completedAt) + ) { + this.#sessions.clear(); } + this.#attribution = attribution; + } + const unreadable: Array<{ session: SessionUsage; error: unknown }> = []; + const homes = new Set([this.#options.codexHome]); + if (this.#options.scanDirectory !== undefined) { + const home = await recordedScanCodexHome(this.#options.scanDirectory); + if (home !== undefined) homes.add(home); + } + // Recovery restores workers to their recorded home; the SDK parent can + // continue in the current home. Apply the same scan membership to both. + const directories = new Set(); + for (const home of homes) { + let directory: string; try { - await readSessionUsage(path, session, this.#options.repository); + directory = await realpath(join(home, "sessions")); } catch (error) { - if (session.threadId === null) throw error; - unreadable.push({ session, error }); + if (isMissingFile(error)) continue; + throw error; + } + if (directories.has(directory)) continue; + directories.add(directory); + for await (const path of sessionFiles(directory)) { + let session = this.#sessions.get(path); + if (session === undefined) { + session = createSessionUsage(); + this.#sessions.set(path, session); + } + try { + await readSessionUsage( + path, + session, + this.#options.repository, + this.#attribution, + ); + } catch (error) { + if (session.threadId === null) throw error; + unreadable.push({ session, error }); + } } } - const included = new Set([this.#threadId, ...this.#receipts.keys()]); - if (this.#options.scanDirectory !== undefined) { + const included = this.#attribution + ? attributedScanThreads(this.#sessions.values(), this.#attribution) + : new Set([this.#threadId, ...this.#receipts.keys()]); + if (!this.#attribution && this.#options.scanDirectory !== undefined) { const scanStartedAt = [...this.#sessions.values()].find( (session) => session.threadId === this.#threadId, @@ -243,7 +333,7 @@ export class ScanCostTracker { } } } - let changed = true; + let changed = this.#attribution === null; while (changed) { changed = false; for (const session of this.#sessions.values()) { @@ -262,7 +352,21 @@ export class ScanCostTracker { if (included.has(session.threadId!)) throw error; } - const usages = new Map(this.#receipts); + let incomplete = false; + const usages = new Map( + [...this.#receipts].filter( + ([threadId]) => + included.has(threadId) && + (!this.#attribution || + this.#attribution.executionThreadIds.includes(threadId)), + ), + ); + if (this.#attribution) { + for (const threadId of included) { + if (!usages.has(threadId)) usages.set(threadId, null); + } + } + const usageSessions = new Map(); for (const [path, tracked] of this.#sessions) { const threadId = tracked.threadId; if (threadId === null || !included.has(threadId)) continue; @@ -274,7 +378,12 @@ export class ScanCostTracker { // Replay only newly associated sessions, including their early events. session = createSessionUsage(); session.events = []; - await readSessionUsage(path, session, this.#options.repository); + await readSessionUsage( + path, + session, + this.#options.repository, + this.#attribution, + ); this.#sessions.set(path, session); } let worker: number | undefined; @@ -282,7 +391,20 @@ export class ScanCostTracker { worker = this.#workers.get(threadId) ?? this.#workers.size + 1; this.#workers.set(threadId, worker); } - for (const event of session.events?.splice(0) ?? []) { + for (const { index, event } of session.events?.splice(0) ?? []) { + let reported = this.#reportedSessionEvents.get(threadId); + if (reported === undefined) { + reported = new Set(); + this.#reportedSessionEvents.set(threadId, reported); + } + // A physical copy keeps each event's position, including repeated + // identical events. Positions count unfiltered records so attribution + // changes can replay the same log without changing occurrence identity. + const identity = `${index}:${createHash("sha256") + .update(JSON.stringify(event)) + .digest("hex")}`; + if (reported.has(identity)) continue; + reported.add(identity); this.#options.onSessionEvent?.({ threadId, parentThreadId: session.parentThreadId, @@ -300,6 +422,22 @@ export class ScanCostTracker { } this.#reportWorkerProgress(session); } + // A copied prefix must not supply model usage for a more complete log. + const previous = usageSessions.get(threadId); + if ( + previous === undefined || + (session.usage?.total_tokens ?? -1) > + (previous.usage?.total_tokens ?? -1) + ) { + usageSessions.set(threadId, session); + } + if ( + session.counterUsage && + session.counterUsage.total_tokens > + (usages.get(threadId)?.total_tokens ?? -1) + ) { + usages.set(threadId, session.counterUsage); + } const receipt = usages.get(threadId); if ( session.usage !== null && @@ -310,19 +448,74 @@ export class ScanCostTracker { ) { usages.set(threadId, session.usage); } + if (!usages.has(threadId)) usages.set(threadId, null); + if ( + (session.counterRegressed && !session.responseUsageObserved) || + session.expectedResponseTokens > session.responseTokens + ) + incomplete = true; + if (session.pendingLineBytes > 0 && !this.#receipts.get(threadId)) + incomplete = true; } let usage: ScanTokenUsage | null = null; for (const value of usages.values()) { if (value === null) { + if (this.#attribution) { + incomplete = true; + continue; + } this.#snapshot = { usage: null, cost: null }; return; } usage = addTokenUsage(usage, value); } - if (usage === null) return; - const cost = estimateScanCost(this.#options.model, usage); - this.#snapshot = { usage, cost }; - this.#reportCost(cost); + if (usage === null) { + this.#snapshot = { usage: null, cost: null }; + return; + } + const modelUsage = new Map(); + let observedModel = false; + for (const [threadId, value] of usages) { + if (value === null) continue; + const session = usageSessions.get(threadId); + for (const [model, tokens] of session?.modelUsage ?? []) { + if (model !== null) observedModel = true; + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, tokens), + ); + } + const remainder = session?.usage + ? subtractTokenUsage(value, session.usage) + : value; + if (remainder !== null && remainder.total_tokens > 0) { + const model = + this.#attribution || (session?.modelUsage.size ?? 0) > 0 + ? null + : (session?.model ?? + (threadId === this.#threadId ? this.#options.model : null)); + modelUsage.set( + model, + addTokenUsage(modelUsage.get(model) ?? null, remainder), + ); + } + } + const reconciled = + observedModel || this.#attribution !== null + ? { + ...usage, + modelUsage: [...modelUsage].map(([model, tokens]) => ({ + model, + ...tokens, + })), + } + : usage; + const measured = incomplete + ? { ...reconciled, coverage: "partial" } + : reconciled; + const cost = estimateScanCost(this.#options.model, measured); + this.#snapshot = { usage: measured, cost }; + this.#reportCost(cost, measured); } #reportWorkerProgress(session: SessionUsage): void { @@ -365,8 +558,17 @@ export class ScanCostTracker { } } - #reportCost(cost: ScanCost | null): void { - if (cost === null) return; + #reportCost(cost: ScanCost | null, usage: unknown): void { + if (cost === null) { + if (this.#options.onCostLowerBound === undefined) return; + const lowerBound = estimateScanCostLowerBound(this.#options.model, usage); + if (lowerBound === null) return; + const signature = JSON.stringify(lowerBound); + if (signature === this.#lastCostLowerBound) return; + this.#lastCostLowerBound = signature; + this.#options.onCostLowerBound(lowerBound); + return; + } const signature = JSON.stringify(cost); if (signature === this.#lastCost) return; this.#lastCost = signature; @@ -396,6 +598,7 @@ async function readSessionUsage( path: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): Promise { if (session.unreadable) return; let file; @@ -417,7 +620,12 @@ async function readSessionUsage( if (bytesRead === 0) return; session.offset += bytesRead; try { - readSessionChunk(buffer.subarray(0, bytesRead), session, repository); + readSessionChunk( + buffer.subarray(0, bytesRead), + session, + repository, + attribution, + ); } catch (error) { session.unreadable = true; session.pendingLine = []; @@ -434,6 +642,7 @@ function readSessionChunk( contents: Buffer, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): void { let lineStart = 0; while (lineStart < contents.length) { @@ -451,13 +660,19 @@ function readSessionChunk( } if (session.pendingLineBytes === 0) { - readSessionEvent(fragment.toString("utf8"), session, repository); + readSessionEvent( + fragment.toString("utf8"), + session, + repository, + attribution, + ); } else { if (fragment.length > 0) session.pendingLine.push(Buffer.from(fragment)); readSessionEvent( Buffer.concat(session.pendingLine, lineBytes).toString("utf8"), session, repository, + attribution, ); session.pendingLine = []; session.pendingLineBytes = 0; @@ -470,6 +685,7 @@ function readSessionEvent( line: string, session: SessionUsage, repository?: string, + attribution: ScanExecutionAttribution | null = null, ): void { if (line.length === 0) return; let event: unknown; @@ -480,10 +696,11 @@ function readSessionEvent( } if (!isRecord(event) || !isRecord(event["payload"])) return; const payload = event["payload"]; + const index = session.eventIndex++; if (event["type"] === "session_meta") { if (session.threadId !== null) { session.replaying = payload["id"] !== session.threadId; - if (!session.replaying) session.events?.push(event); + if (!session.replaying) session.events?.push({ index, event }); return; } if (typeof payload["id"] === "string") { @@ -492,9 +709,10 @@ function readSessionEvent( if (typeof payload["cwd"] === "string") { session.workingDirectory = payload["cwd"]; } + if (typeof payload["model"] === "string") session.model = payload["model"]; session.startedAt = sessionStartedAt(payload["timestamp"]); session.parentThreadId = sessionParentThreadId(payload); - session.events?.push(event); + session.events?.push({ index, event }); return; } if (session.replaying) { @@ -515,12 +733,87 @@ function readSessionEvent( : turnOrder !== null && turnOrder >= threadOrder; if (owned) { session.replaying = false; - session.events?.push(event); + session.events?.push({ index, event }); } } return; } - session.events?.push(event); + if ( + (event["type"] === "turn_context" || payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + session.currentTurnId = payload["turn_id"]; + } + if ( + event["type"] === "turn_context" && + typeof payload["model"] === "string" + ) { + session.model = payload["model"]; + } + if (event["type"] === "token_usage_record") { + const responseId = payload["response_id"]; + const usage = tokenUsage(payload["usage"]); + if ( + typeof responseId !== "string" || + usage === null || + (typeof payload["thread_id"] === "string" && + payload["thread_id"] !== session.threadId) || + session.responseIds.has(responseId) + ) + return; + session.responseIds.add(responseId); + const cumulative = tokenUsage(payload["thread_token_usage"]); + if (cumulative) + session.expectedResponseTokens = Math.max( + session.expectedResponseTokens, + cumulative.total_tokens, + ); + if (!session.responseUsageObserved) { + // Exact receipts include compaction and survive counter resets. Keep the + // legacy counter as an independent lower bound, never add it to receipts. + session.responseUsageObserved = true; + session.usage = null; + session.modelUsage.clear(); + } + session.responseTokens += usage.total_tokens; + const turnId = + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : session.currentTurnId; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + turnId, + event["timestamp"], + ) + ) + return; + const model = + typeof payload["model"] === "string" ? payload["model"] : session.model; + session.usage = addTokenUsage(session.usage, usage); + session.modelUsage.set( + model, + addTokenUsage(session.modelUsage.get(model) ?? null, usage), + ); + session.events?.push({ index, event }); + return; + } + const attributable = + attribution === null || + isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ); + if (attributable) session.events?.push({ index, event }); + if ( + !attributable && + !(event["type"] === "event_msg" && payload["type"] === "token_count") + ) + return; if (event["type"] === "response_item") { session.progress.push(...sessionProgressUpdates(payload)); if (repository === undefined) return; @@ -658,7 +951,41 @@ function readSessionEvent( session.inheritedUsage === null ? usage : subtractTokenUsage(usage, session.inheritedUsage); - if (ownUsage !== null) session.usage = ownUsage; + if (ownUsage !== null) { + const delta = + session.previousUsage === null + ? ownUsage + : subtractTokenUsage(ownUsage, session.previousUsage); + if ( + session.previousUsage !== null && + ownUsage.total_tokens < session.previousUsage.total_tokens + ) { + session.counterRegressed = true; + return; + } + session.previousUsage = ownUsage; + if ( + attribution && + !isAttributedScanEvent( + attribution, + session.threadId!, + session.currentTurnId, + event["timestamp"], + ) + ) + return; + if (delta !== null && !session.responseUsageObserved) { + session.modelUsage.set( + session.model, + addTokenUsage(session.modelUsage.get(session.model) ?? null, delta), + ); + } + session.counterUsage = + attribution && delta !== null + ? addTokenUsage(session.counterUsage, delta) + : ownUsage; + if (!session.responseUsageObserved) session.usage = session.counterUsage; + } } function uuid7Order(value: unknown): bigint | null { diff --git a/sdk/typescript/src/custom-validation-prompt.ts b/sdk/typescript/src/custom-validation-prompt.ts index 93a1ec42e..4cabb3f1a 100644 --- a/sdk/typescript/src/custom-validation-prompt.ts +++ b/sdk/typescript/src/custom-validation-prompt.ts @@ -9,7 +9,7 @@ import { PLUGIN_NAME } from "./runtime.js"; // the ordinary validation sequence with a custom-validation request. const SOURCES = { "references/core-scan.md": - "77b082eb8613cf93427ff730e4ae5d85b0a0dca37c02a8af1ea69f679ac3d1d9", + "a489a22c84f041a9cfbfccc69b47a0dd2a4d5e7113dbcb40301ab0a793aa6379", "skills/security-scan/SKILL.md": "5b8f5d7debeca14c6b37e8e7ba737671362b8eb4b7f49e693c99c6bd04bc8fa0", "skills/security-diff-scan/SKILL.md": diff --git a/sdk/typescript/src/deep-scan-finalization.ts b/sdk/typescript/src/deep-scan-finalization.ts new file mode 100644 index 000000000..e18bf9f58 --- /dev/null +++ b/sdk/typescript/src/deep-scan-finalization.ts @@ -0,0 +1,26 @@ +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +type WorkbenchRunner = (args: string[]) => Promise>; + +/** Load the same publisher used by native Deep from the installed plugin. */ +export async function resumeSelectedDeepScan(input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; +}): Promise { + const helper = ( + await import(pathToFileURL(join(input.pluginRoot, "mcp/helpers.mjs")).href) + ).default as { + resumeSelectedDeepScan: (input: { + scanId: string; + threadId: string; + pluginRoot: string; + runWorkbench: WorkbenchRunner; + signal: AbortSignal; + }) => Promise; + }; + await helper.resumeSelectedDeepScan(input); +} diff --git a/sdk/typescript/src/preflight-config.ts b/sdk/typescript/src/preflight-config.ts new file mode 100644 index 000000000..5c1c59cf9 --- /dev/null +++ b/sdk/typescript/src/preflight-config.ts @@ -0,0 +1,142 @@ +import { isAbsolute } from "node:path"; +import { + EXTERNAL_CODEX_PROVIDERS, + isExternalModelProvider, + resolveCodexProfile, + scanModelProvider, + type JsonObject, +} from "./config.js"; + +export function scanPreflightCodexConfig(config: JsonObject): JsonObject { + const safeString = (value: unknown): value is string => + typeof value === "string" && + value.length > 0 && + !/[\u0000-\u001f\u007f]/u.test(value); + const safeProfileName = (value: unknown): value is string => + safeString(value) && /^[A-Za-z0-9_-]+$/u.test(value); + const safeInteger = (value: unknown): value is number => + typeof value === "number" && Number.isSafeInteger(value) && value >= 0; + const capabilityFeatures = (value: unknown): JsonObject => { + if (!isRecord(value)) return {}; + const result: JsonObject = {}; + for (const key of ["goals", "multi_agent", "enable_fanout"]) { + if (typeof value[key] === "boolean") result[key] = value[key]; + } + const multiAgent = value["multi_agent_v2"]; + if (typeof multiAgent === "boolean") { + result["multi_agent_v2"] = multiAgent; + } else if (isRecord(multiAgent)) { + const sanitized: JsonObject = {}; + if (typeof multiAgent["enabled"] === "boolean") { + sanitized["enabled"] = multiAgent["enabled"]; + } + const capacity = multiAgent["max_concurrent_threads_per_session"]; + if (safeInteger(capacity)) { + sanitized["max_concurrent_threads_per_session"] = capacity; + } + if (Object.keys(sanitized).length > 0) { + result["multi_agent_v2"] = sanitized; + } + } + return result; + }; + const executionConfig = (source: JsonObject): JsonObject => { + const result: JsonObject = {}; + for (const key of [ + "model", + "model_reasoning_effort", + "model_reasoning_summary", + "model_provider", + "service_tier", + ]) { + const value = source[key]; + if (safeString(value)) result[key] = value; + } + const features = capabilityFeatures(source["features"]); + if (Object.keys(features).length > 0) result["features"] = features; + const agents = source["agents"]; + if (isRecord(agents)) { + const sanitized: JsonObject = {}; + for (const key of ["max_threads", "max_depth"]) { + const value = agents[key]; + if (safeInteger(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) result["agents"] = sanitized; + } + const multiagent = source["multiagent_config"]; + if (isRecord(multiagent) && safeInteger(multiagent["max_concurrency"])) { + result["multiagent_config"] = { + max_concurrency: multiagent["max_concurrency"], + }; + } + return result; + }; + const result = executionConfig(config); + // Keep the effective summary even when preflight filters the profile name. + const reasoningSummary = + resolveCodexProfile(config)["model_reasoning_summary"]; + if (safeString(reasoningSummary)) { + result["model_reasoning_summary"] = reasoningSummary; + } + const selectedProfile = safeProfileName(config["profile"]) + ? config["profile"] + : undefined; + if (selectedProfile !== undefined) { + result["profile"] = selectedProfile; + } + const profiles = config["profiles"]; + if (isRecord(profiles)) { + const sanitized: JsonObject = {}; + for (const [name, profile] of Object.entries(profiles)) { + if (!safeProfileName(name) || !isRecord(profile)) continue; + const projected = executionConfig(profile as JsonObject); + if (Object.keys(projected).length === 0) continue; + sanitized[name] = projected; + } + if (Object.keys(sanitized).length > 0) result["profiles"] = sanitized; + } + const modelProvider = scanModelProvider(result); + if (isExternalModelProvider(modelProvider)) { + result["model_providers"] = { + [modelProvider]: { ...EXTERNAL_CODEX_PROVIDERS[modelProvider] }, + }; + } else if (modelProvider === "amazon-bedrock") { + const providers = config["model_providers"]; + const provider = isRecord(providers) ? providers[modelProvider] : undefined; + const aws = isRecord(provider) ? provider["aws"] : undefined; + if (isRecord(aws)) { + const sanitized: JsonObject = {}; + for (const key of ["region", "profile"]) { + const value = aws[key]; + if (safeString(value)) sanitized[key] = value; + } + if (Object.keys(sanitized).length > 0) { + result["model_providers"] = { + [modelProvider]: { aws: sanitized }, + }; + } + } + } + const rootMarkers = config["project_root_markers"]; + if (Array.isArray(rootMarkers)) { + result["project_root_markers"] = rootMarkers.filter(safeString); + } + const projects = config["projects"]; + if (isRecord(projects)) { + const sanitized: JsonObject = {}; + for (const [path, project] of Object.entries(projects)) { + if (!safeString(path) || !isAbsolute(path) || !isRecord(project)) { + continue; + } + const trust = project["trust_level"]; + if (trust !== "trusted" && trust !== "untrusted") continue; + sanitized[path] = { trust_level: trust }; + } + if (Object.keys(sanitized).length > 0) result["projects"] = sanitized; + } + return result; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/sdk/typescript/src/reasoning-summary.ts b/sdk/typescript/src/reasoning-summary.ts new file mode 100644 index 000000000..a313bc00c --- /dev/null +++ b/sdk/typescript/src/reasoning-summary.ts @@ -0,0 +1,69 @@ +import { + inlineToml, + resolveCodexProfile, + scanModelConfiguration, + type JsonObject, +} from "./config.js"; +import { scanPreflightCodexConfig } from "./preflight-config.js"; +import { + runCodexCommand, + type CodexCommand, + type ProcessEnvironment, +} from "./runtime.js"; + +/** Capture a future owner's selected model default; never infer historical settings. */ +export async function captureOriginalReasoningSummary(options: { + config: JsonObject; + command: CodexCommand; + cwd: string; + environment: ProcessEnvironment; + signal: AbortSignal; +}): Promise { + // Keep explicit selections and invalid values with their existing native validator. + if ( + scanPreflightCodexConfig(options.config)["model_reasoning_summary"] !== + undefined || + resolveCodexProfile(options.config)["model_reasoning_summary"] !== undefined + ) + return undefined; + const { model } = scanModelConfiguration(options.config); + // The same per-session overrides protect the lookup from concurrent home edits. + const config = JSON.parse(JSON.stringify(options.config)) as JsonObject; + const args = [ + "debug", + "models", + ...Object.entries(config).flatMap(([key, value]) => [ + "--config", + `${key}=${inlineToml(value)}`, + ]), + ]; + const result = await runCodexCommand( + options.command, + args, + options.environment, + undefined, + options.signal, + options.cwd, + ); + // Older native executables and models absent from their catalog provide no + // recoverable value. Preserve omission instead of choosing another default. + if (!result.success) return undefined; + let catalog: unknown; + try { + catalog = JSON.parse(result.stdout); + } catch { + return undefined; + } + if (!isRecord(catalog) || !Array.isArray(catalog["models"])) return undefined; + const selected = catalog["models"].find( + (entry: unknown) => isRecord(entry) && entry["slug"] === model, + ); + return isRecord(selected) && + typeof selected["default_reasoning_summary"] === "string" + ? selected["default_reasoning_summary"] + : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts index 8d2d2a442..f85477f8a 100644 --- a/sdk/typescript/src/runtime.ts +++ b/sdk/typescript/src/runtime.ts @@ -2702,12 +2702,14 @@ export async function runCodexCommand( environment: ProcessEnvironment, input?: string | Uint8Array, signal?: AbortSignal, + cwd?: string, ): Promise { const child = spawn(executablePathForSpawn(command.command), [...args], { env: environment, stdio: ["pipe", "pipe", "pipe"], windowsHide: true, signal, + ...(cwd === undefined ? {} : { cwd }), }); let stdout = ""; let stderr = ""; diff --git a/sdk/typescript/src/scan-logs.ts b/sdk/typescript/src/scan-logs.ts index 851a8f2da..f9976cc65 100644 --- a/sdk/typescript/src/scan-logs.ts +++ b/sdk/typescript/src/scan-logs.ts @@ -5,9 +5,13 @@ import { sessionFiles } from "./cost.js"; import { CodexSecurityError } from "./errors.js"; import type { JsonObject } from "./config.js"; import { + attributedScanThreads, + isAttributedScanEvent, isScanArtifactDirectory, + recordedScanCodexHome, sessionParentThreadId, sessionStartedAt, + type ScanExecutionAttribution, } from "./scan-sessions.js"; interface ScanLogOptions { @@ -19,6 +23,7 @@ interface ScanLogOptions { scanDirectory?: string; completedAt?: string | null; allowMissingRoot?: boolean; + executionAttribution?: ScanExecutionAttribution | null; } export type ScanLogSource = JsonObject & { @@ -26,6 +31,7 @@ export type ScanLogSource = JsonObject & { continuationThreadId?: string; threadIds?: string[]; executionThreadIds?: string[]; + executionAttribution?: ScanExecutionAttribution | null; mode?: string; scanDir?: string; progress?: { status?: string; updatedAt?: string }; @@ -47,6 +53,7 @@ export function readSavedScanLogs( threadId: threadId ?? scan.threadIds?.[0], threadIds: scan.threadIds, executionThreadIds: scan.executionThreadIds ?? [], + executionAttribution: scan.executionAttribution, codexHome, allowMissingRoot: options.allowMissingRoot, scanDirectory: scan.mode === "deep" ? scan.scanDir : undefined, @@ -106,15 +113,26 @@ export async function findScanSession( export async function readScanLogs(options: ScanLogOptions) { const logs = new Map(); + const sessionPaths = new Map(); const homes = new Set( typeof options.codexHome === "string" ? [options.codexHome] : options.codexHome, ); + if (options.scanDirectory !== undefined) { + const home = await recordedScanCodexHome(options.scanDirectory); + if (home !== undefined) homes.add(home); + } + // Recovered workers retain their original home; the parent can use the current home. for (const directory of ["sessions", "archived_sessions"]) { for (const home of homes) { for await (const session of scanSessions(home, directory)) { - if (!logs.has(session.threadId)) logs.set(session.threadId, session); + const paths = sessionPaths.get(session.threadId); + if (paths) paths.push(session.path); + else { + logs.set(session.threadId, session); + sessionPaths.set(session.threadId, [session.path]); + } } } } @@ -126,15 +144,20 @@ export async function readScanLogs(options: ScanLogOptions) { ); } - const included = new Set([ - ...(options.threadId ? [options.threadId] : []), - ...(options.threadIds ?? []), - ...(options.executionThreadIds ?? []), - ]); + const attribution = options.executionAttribution?.legacy + ? null + : options.executionAttribution; + const included = attribution + ? attributedScanThreads(logs.values(), attribution) + : new Set([ + ...(options.threadId ? [options.threadId] : []), + ...(options.threadIds ?? []), + ...(options.executionThreadIds ?? []), + ]); // A Desktop owner can contain other work. Include its log without treating // the whole conversation tree as part of this scan. const traversed = new Set(options.executionThreadIds ?? included); - const pending = [...traversed]; + const pending = attribution ? [] : [...traversed]; for (const parentId of pending) { const parent = logs.get(parentId); for (const session of logs.values()) { @@ -155,29 +178,23 @@ export async function readScanLogs(options: ScanLogOptions) { const sessions: SessionLog[] = []; for (const threadId of included) { const session = logs.get(threadId); - if (session !== undefined) sessions.push(session); + if (session !== undefined) { + // Compare only scan-owned logs. Keep traversal order unless a later copy + // contains every attributed event followed by additional events. + for (const path of sessionPaths.get(threadId)!.slice(1)) { + if (await extendsSessionLog(path, session, attribution)) + session.path = path; + } + sessions.push(session); + } } const events: Record[] = []; for (const session of sessions) { - let replaying = false; - for await (const event of sessionEvents(session.path)) { - const payload = event["payload"]; - if (event["type"] === "session_meta" && isRecord(payload)) { - replaying = payload["id"] !== session.threadId; - } - if (replaying) { - if ( - event["type"] !== "event_msg" || - !isRecord(payload) || - payload["type"] !== "task_started" || - typeof payload["started_at"] !== "number" || - session.startedAt === null || - payload["started_at"] < Math.floor(session.startedAt / 1_000) - ) { - continue; - } - replaying = false; - } + for await (const event of attributedSessionEvents( + session.path, + session, + attribution, + )) { events.push({ threadId: session.threadId, event }); } } @@ -194,6 +211,81 @@ export async function readScanLogs(options: ScanLogOptions) { }; } +async function* attributedSessionEvents( + path: string, + session: SessionLog, + attribution: ScanExecutionAttribution | null | undefined, +): AsyncGenerator> { + let replaying = false; + let turnId: string | null = null; + for await (const event of sessionEvents(path)) { + const payload = event["payload"]; + if ( + isRecord(payload) && + (event["type"] === "turn_context" || + payload["type"] === "task_started") && + typeof payload["turn_id"] === "string" + ) { + turnId = payload["turn_id"]; + } + if (event["type"] === "session_meta" && isRecord(payload)) { + replaying = payload["id"] !== session.threadId; + } + if (replaying) { + if ( + event["type"] !== "event_msg" || + !isRecord(payload) || + payload["type"] !== "task_started" || + typeof payload["started_at"] !== "number" || + session.startedAt === null || + payload["started_at"] < Math.floor(session.startedAt / 1_000) + ) { + continue; + } + replaying = false; + } + if ( + !attribution || + event["type"] === "session_meta" || + isAttributedScanEvent( + attribution, + session.threadId, + event["type"] === "token_usage_record" && + isRecord(payload) && + typeof payload["turn_id"] === "string" + ? payload["turn_id"] + : turnId, + event["timestamp"], + ) + ) { + yield event; + } + } +} + +async function extendsSessionLog( + path: string, + session: SessionLog, + attribution: ScanExecutionAttribution | null | undefined, +): Promise { + const previous = attributedSessionEvents(session.path, session, attribution); + try { + for await (const event of attributedSessionEvents( + path, + session, + attribution, + )) { + const recorded = await previous.next(); + if (recorded.done) return true; + if (JSON.stringify(event) !== JSON.stringify(recorded.value)) + return false; + } + return false; + } finally { + await previous.return(undefined); + } +} + function belongsToScan( session: SessionLog, root: SessionLog, diff --git a/sdk/typescript/src/scan-sessions.ts b/sdk/typescript/src/scan-sessions.ts index 676f7a10b..1cbfa1096 100644 --- a/sdk/typescript/src/scan-sessions.ts +++ b/sdk/typescript/src/scan-sessions.ts @@ -1,5 +1,99 @@ +import { readFile } from "node:fs/promises"; import { isAbsolute, join, relative, sep } from "node:path"; +export async function recordedScanCodexHome( + scanDirectory: string, +): Promise { + try { + const saved: unknown = JSON.parse( + await readFile( + join( + scanDirectory, + "artifacts", + "deep_discovery", + "execution-settings.json", + ), + "utf8", + ), + ); + if ( + isRecord(saved) && + saved["version"] === 1 && + isRecord(saved["settings"]) + ) { + const home = saved["settings"]["codexHome"]; + if (typeof home === "string" && home !== "") return home; + } + } catch (error) { + if (!isRecord(error) || error["code"] !== "ENOENT") throw error; + } + return undefined; +} + +export interface ScanExecutionAttribution { + formatVersion: 1; + legacy?: true; + executionThreadIds: string[]; + owner: { + threadId: string | null; + turnId: string | null; + startedAt: string; + dedicated?: boolean; + }; + startedAt: string; + completedAt: string | null; +} + +export function attributedScanThreads( + sessions: Iterable<{ + threadId: string | null; + parentThreadId: string | null; + }>, + attribution: ScanExecutionAttribution, +): Set { + const included = new Set(attribution.executionThreadIds); + const pending = [...included]; + const all = [...sessions]; + for (const parent of pending) { + for (const session of all) { + if ( + session.threadId !== null && + session.parentThreadId === parent && + !included.has(session.threadId) + ) { + included.add(session.threadId); + pending.push(session.threadId); + } + } + } + if (attribution.owner.threadId) included.add(attribution.owner.threadId); + return included; +} + +export function isAttributedScanEvent( + attribution: ScanExecutionAttribution, + threadId: string, + turnId: string | null, + timestamp: unknown, +): boolean { + const time = sessionStartedAt(timestamp); + if ( + time === null || + time < Date.parse(attribution.startedAt) || + (attribution.completedAt !== null && + time > Date.parse(attribution.completedAt)) + ) + return false; + if ( + threadId !== attribution.owner.threadId || + attribution.executionThreadIds.includes(threadId) + ) + return true; + return ( + attribution.owner.turnId !== null && turnId === attribution.owner.turnId + ); +} + export function sessionStartedAt(timestamp: unknown): number | null { const startedAt = typeof timestamp === "string" ? Date.parse(timestamp) : Number.NaN; diff --git a/sdk/typescript/tests-ts/api-audit-admission.test.ts b/sdk/typescript/tests-ts/api-audit-admission.test.ts new file mode 100644 index 000000000..0bfc8c269 --- /dev/null +++ b/sdk/typescript/tests-ts/api-audit-admission.test.ts @@ -0,0 +1,395 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import { build } from "esbuild"; +import { runScanEvents } from "../src/api.js"; +import type { ScanDraftInput } from "../src/accepted-audit.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; +import { + completedEvents, + createApiTestFixtures, +} from "./support/api-events.js"; + +const { temporaryDirectory, copyCompletedScan, cleanup } = + createApiTestFixtures(); +afterEach(cleanup); + +const bundle = await build({ + stdin: { + resolveDir: fileURLToPath( + new URL("../../../plugins/codex-security/mcp-app/", import.meta.url), + ), + contents: `export * from "./src/artifact-scan-draft.ts"; + export * from "./src/deep-scan/artifacts.ts"; + export * from "./src/deep-scan/artifact-validation.ts"; + export * from "./src/deep-scan/worker-runner.ts";`, + }, + bundle: true, + format: "esm", + platform: "node", + loader: { ".md": "text" }, + write: false, +}); +const bundlePath = join(await temporaryDirectory(), "deep-admission.mjs"); +await writeFile(bundlePath, bundle.outputFiles[0]!.contents); +const { + createDeepScanArtifacts, + recordCodexSecurityScanDraft, + readDiscoveryAuditDraft, + DeepScanWorkerRunner, +} = await import(pathToFileURL(bundlePath).href); + +const scanId = "811aef98-3709-4c2d-8b7a-742977521865"; +type Mutation = + | "missing-findings" + | "contradictory-coverage" + | "inverted-lines" + | "wrong-scan" + | "legacy-details"; +const cases: { + name: string; + coverage: "complete" | "partial" | "unknown"; + complete?: boolean; + mutation?: Mutation; + accepted: boolean; +}[] = [ + { + name: "complete coverage", + coverage: "complete", + complete: true, + accepted: true, + }, + { + name: "partial coverage", + coverage: "partial", + complete: true, + accepted: true, + }, + { + name: "unknown coverage", + coverage: "unknown", + complete: true, + accepted: true, + }, + { name: "omitted completion marker", coverage: "partial", accepted: true }, + { + name: "persisted legacy details", + coverage: "partial", + mutation: "legacy-details", + accepted: true, + }, + { + name: "unfinished checkpoint", + coverage: "partial", + complete: false, + accepted: false, + }, + { + name: "missing findings", + coverage: "partial", + mutation: "missing-findings", + accepted: false, + }, + { + name: "complete coverage with deferred work", + coverage: "partial", + mutation: "contradictory-coverage", + accepted: false, + }, + { + name: "inverted finding lines", + coverage: "partial", + mutation: "inverted-lines", + accepted: false, + }, + { + name: "mismatched canonical scan ID", + coverage: "partial", + mutation: "wrong-scan", + accepted: false, + }, +]; + +for (const scenario of cases) { + test(`Standard and Deep production admission: ${scenario.name}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const standardRoot = join(root, "standard"); + const deepRoot = join(root, "deep"); + await Promise.all([ + mkdir(repository), + mkdir(standardRoot, { mode: 0o700 }), + mkdir(deepRoot, { mode: 0o700 }), + ]); + const semantic: ScanDraftInput = { + scanId, + ...(scenario.complete === undefined + ? {} + : { complete: scenario.complete }), + scope: { summary: "Archive extraction." }, + threatModel: { summary: "An untrusted caller supplies archive entries." }, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: "An archive entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { level: "high", rationale: "Source review." }, + taxonomy: { category: "path-traversal", cwe: ["CWE-22"] }, + locations: [{ path: "extract.py", startLine: 4, endLine: 7 }], + remediation: "Validate the resolved output path before writing.", + provenance: { source: "local_plugin", candidateId: "archive-entry" }, + }, + ], + coverage: { + completeness: scenario.coverage, + surfaces: [{ label: "Archive extraction", disposition: "reported" }], + explicitExclusions: [], + deferred: + scenario.coverage === "complete" + ? [] + : [ + { + id: "deployment", + reason: "Deployment controls remain unverified.", + }, + ], + }, + }; + await recordCodexSecurityScanDraft( + { + root: standardRoot, + repoRoot: repository, + layout: "scan", + scanId, + mode: "standard", + status: "running", + scope: ".", + targetContract: { + target: { + allowedKinds: ["directory_snapshot"], + targetId: "target_example", + displayName: "example", + requiredSnapshotDigest: `codex-security-snapshot/v1:sha256:${"a".repeat(64)}`, + }, + scope: { requiredIncludePaths: ["."], requiredExcludePaths: [] }, + diffTarget: null, + }, + }, + semantic, + ); + const submitted = mutateDraft(semantic, scenario.mutation); + const findings = { + scanId: submitted.scanId, + findings: submitted.findings?.map((finding) => ({ + ...finding, + findingId: "finding_example", + occurrenceId: "occurrence_example", + fingerprints: { identity: "synthetic" }, + })), + }; + const coverage = JSON.parse( + await readFile(join(standardRoot, "coverage.json"), "utf8"), + ); + Object.assign(coverage, submitted.coverage); + await Promise.all([ + writeFile(join(standardRoot, "findings.json"), JSON.stringify(findings)), + writeFile(join(standardRoot, "coverage.json"), JSON.stringify(coverage)), + ]); + const standard = await observeStandardAdmission( + root, + repository, + standardRoot, + scanId, + ); + + const artifacts = createDeepScanArtifacts(deepRoot); + const acceptedPaths: string[] = []; + let executions = 0; + const runner = new DeepScanWorkerRunner({ + run: { + scanId, + scanDir: deepRoot, + targetPath: repository, + scope: ".", + config: { subagents: 0 }, + }, + artifacts, + pluginRoot: PLUGIN_ROOT, + signal: new AbortController().signal, + retryDelaysMs: [], + random: () => 0, + log: () => {}, + clock: { now: () => Date.now(), sleep: async () => {} }, + executor: { + async run(request: { + artifactContext: { root: string }; + onThreadStarted?: (id: string) => Promise; + }) { + executions++; + await request.onThreadStarted?.("deep-thread"); + await writeFile( + join(request.artifactContext.root, "result.json"), + JSON.stringify(submitted), + ); + return { threadId: "deep-thread" }; + }, + }, + store: { + async updateWorker(update: { + status: string; + resultManifestPath?: string; + }) { + if (update.status === "succeeded") { + acceptedPaths.push(update.resultManifestPath!); + return { ...update, completionSequence: 1 }; + } + return update; + }, + }, + }); + const deepResult = await runner.runDiscoveryWorker( + "worker-1", + "discovery-1", + ); + expect(executions).toBe(1); + if (scenario.accepted) { + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + expect(deepResult.status).toBe("succeeded"); + expect(acceptedPaths).toEqual([deepResult.worker.resultPath]); + const deepDraft: ScanDraftInput = await readDiscoveryAuditDraft( + artifacts, + deepResult.worker.resultPath, + scanId, + ); + expect(standard.drafts[0]!.findings).toEqual(deepDraft.findings); + expect(standard.drafts[0]!.coverage).toEqual(deepDraft.coverage); + expect(standard.drafts[0]!.scope).toEqual(deepDraft.scope); + expect(standard.drafts[0]!.threatModel).toEqual(deepDraft.threatModel); + if (scenario.mutation === "legacy-details") { + expect(deepDraft.findings[0]!["validation"]).toEqual({ + limitations: ["Legacy persisted limitation."], + }); + } + } else { + expect(standard.error).toBeInstanceOf(Error); + expect(standard.error).not.toBe(standard.finalization); + expect(standard.finalizations).toBe(0); + expect(deepResult.status).toBe("failed"); + expect(acceptedPaths).toHaveLength(0); + } + expect(standard.calls).toBe(1); + const manifest = JSON.parse( + await readFile(join(standardRoot, "scan-manifest.json"), "utf8"), + ); + expect(manifest.scan.sealedAt).toBeUndefined(); + await expect(readFile(join(standardRoot, "report.md"))).rejects.toThrow(); + }); +} + +test("Standard admission preserves existing canonical scan IDs", async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + await mkdir(repository); + const scanDir = await copyCompletedScan(root); + const manifest = JSON.parse( + await readFile(join(scanDir, "scan-manifest.json"), "utf8"), + ); + const standard = await observeStandardAdmission(root, repository, scanDir); + expect(manifest.scan.id).toBe("scan_example_001"); + expect(standard.error).toBe(standard.finalization); + expect(standard.finalizations).toBe(1); + expect(standard.drafts).toHaveLength(1); + expect(standard.calls).toBe(1); + expect(standard.drafts[0]!.scanId).toBe(manifest.scan.id); +}); + +async function observeStandardAdmission( + root: string, + repository: string, + scanDir: string, + scanId?: string, +) { + const pluginRoot = join(root, "observed-plugin"); + const helperPath = join(pluginRoot, "mcp", "helpers.mjs"); + await mkdir(join(pluginRoot, "mcp"), { recursive: true }); + // The bundled exports are immutable getters. Observe the real parser through + // a local delegator instead of mocking its module or copying its semantics. + await writeFile( + helperPath, + `import helpers from ${JSON.stringify(pathToFileURL(join(PLUGIN_ROOT, "mcp", "helpers.mjs")).href)}; +export const drafts = []; +export let calls = 0; +export default { ...helpers, parseCanonicalScanDraft(input) { + calls++; + const draft = helpers.parseCanonicalScanDraft(input); + drafts.push(draft); + return draft; +} };`, + ); + const observed: { drafts: ScanDraftInput[]; calls: number } = await import( + pathToFileURL(helperPath).href + ); + const finalization = new Error("The enclosing finalizer owns the next step."); + let finalizations = 0; + const error = await runScanEvents({ + scanId, + thread: { + id: "standard-thread", + async runStreamed() { + return { events: completedEvents("standard-thread") }; + }, + }, + events: completedEvents("standard-thread"), + signal: new AbortController().signal, + scanDir, + pluginRoot, + expectation: { + repository, + repositoryRevision: null, + target: { kind: "repository", paths: [] }, + mode: "standard", + pluginVersion: "0.1.0", + }, + onFinalize: async () => { + finalizations++; + throw finalization; + }, + }).catch((error: unknown) => error); + return { + error, + finalization, + finalizations, + drafts: observed.drafts, + calls: observed.calls, + }; +} + +function mutateDraft( + input: ScanDraftInput, + mutation?: Mutation, +): ScanDraftInput { + const draft = structuredClone(input); + if (mutation === "missing-findings") + return { ...draft, findings: undefined } as unknown as ScanDraftInput; + if (mutation === "wrong-scan") + draft.scanId = "553a0c18-dcdf-4a3b-8e39-2751a8187bce"; + if (mutation === "contradictory-coverage") + draft.coverage["completeness"] = "complete"; + if (mutation === "inverted-lines") { + const locations = draft.findings[0]!["locations"] as Record< + string, + unknown + >[]; + locations[0]!["endLine"] = 1; + } + if (mutation === "legacy-details") + draft.findings[0]!["validation"] = { + method: null, + limitations: "Legacy persisted limitation.", + }; + return draft; +} diff --git a/sdk/typescript/tests-ts/api-events.test.ts b/sdk/typescript/tests-ts/api-events.test.ts index 28d9b1212..2f7d79c73 100644 --- a/sdk/typescript/tests-ts/api-events.test.ts +++ b/sdk/typescript/tests-ts/api-events.test.ts @@ -1,4 +1,4 @@ -import { mkdir, stat } from "node:fs/promises"; +import { mkdir, rm, stat } from "node:fs/promises"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { @@ -477,7 +477,8 @@ describe("one-shot scan events", () => { test("lets the workbench seal artifacts before validating completed scans", async () => { const root = await temporaryDirectory(); - const scanDir = join(root, "scan"); + const scanDir = await copyCompletedScan(root); + await rm(join(scanDir, "report.md")); const events = completedEvents(); let finalized = false; @@ -506,7 +507,8 @@ describe("one-shot scan events", () => { cache_write_input_tokens: 0, output_tokens: 3, }); - expect(existsSync(join(scanDir, "scan-manifest.json"))).toBe(false); + expect(existsSync(join(scanDir, "scan-manifest.json"))).toBe(true); + expect(existsSync(join(scanDir, "report.md"))).toBe(false); await copyCompletedScan(root); finalized = true; }, diff --git a/sdk/typescript/tests-ts/api-policy.test.ts b/sdk/typescript/tests-ts/api-policy.test.ts index 08d057f01..899ffc59e 100644 --- a/sdk/typescript/tests-ts/api-policy.test.ts +++ b/sdk/typescript/tests-ts/api-policy.test.ts @@ -1532,6 +1532,93 @@ describe("CodexSecurity policy API", () => { await f.security.close(); }); + test.each(["architecture", "threat_model"] as const)( + "enforces priced policy usage despite an unpriced remainder in %s", + async (crossingStage) => { + const costs: number[] = []; + const f = await setup({ + config: { codexOverrides: { model: "gpt-5.6-sol" } }, + stream: async function* (stage, signal) { + if (stage !== crossingStage) { + yield* events(stage); + return; + } + const directory = join(f.root, "codex-home", "sessions"); + await mkdir(directory, { recursive: true }); + const thread = `policy-${stage}`; + const input = stage === "architecture" ? 1_200 : 1_000; + for (const [id, model, parent, inputTokens] of [ + [thread, "gpt-5.6-sol", undefined, input], + ["unpriced-policy-worker", "synthetic-unpriced-model", thread, 100], + ] as const) { + await writeFile( + join(directory, `${id}.jsonl`), + [ + JSON.stringify({ + type: "session_meta", + payload: { + id, + ...(parent === undefined + ? {} + : { parent_thread_id: parent }), + }, + }), + JSON.stringify({ type: "turn_context", payload: { model } }), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + output_tokens: 0, + }, + }, + }, + }), + "", + ].join("\n"), + ); + } + yield { type: "thread.started", thread_id: thread }; + await new Promise((resolve) => { + if (signal.aborted) resolve(); + else + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + throw signal.reason; + }, + }); + const keepAlive = setTimeout(() => {}, 10_000); + try { + await expect( + f.security.generatePolicy(f.repository, { + outputDir: f.outputDir, + maxCostUsd: 0.0045, + signal: AbortSignal.timeout(5_000), + onCost: (cost) => costs.push(cost.estimatedUsd), + }), + ).rejects.toThrow("exceeded its $0.0045 cost limit"); + expect(f.threads).toHaveLength( + crossingStage === "architecture" ? 1 : 2, + ); + if (crossingStage === "architecture") expect(costs).toEqual([]); + else { + expect(costs.length).toBeGreaterThan(0); + for (const cost of costs) expect(cost).toBeCloseTo(0.0006, 12); + } + if (crossingStage === "threat_model") { + expect( + await readFile(join(f.outputDir, "project-spec.md"), "utf8"), + ).toContain("src/service.ts:1"); + } + } finally { + clearTimeout(keepAlive); + await f.security.close(); + } + }, + ); + test("enforces one cost budget across stages and preserves completed evidence", async () => { const f = await setup(); await expect( diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 68fa92e1a..1d96a821b 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -4767,6 +4767,183 @@ describe("CodexSecurity orchestration", () => { }, ); + test.each([ + ["standard", false], + ["deep", false], + ["standard", true], + ["deep", true], + ] as const)( + "enforces priced usage with an unpriced remainder (%s, raised limit: %j)", + async (mode, raised) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const commands: Array = []; + const costs: number[] = []; + let turns = 0; + let releaseIncrease!: () => void; + const increased = new Promise((resolve) => { + releaseIncrease = resolve; + }); + const knownUsage = { + input_tokens: raised ? 2_500 : 1_250, + cached_input_tokens: 200, + output_tokens: 30, + }; + const expectedCost = estimateScanCost("gpt-5.6-sol", knownUsage)!; + const writePricedUsage = async (usage: Record) => { + const path = await writeUsageSession(codexHome, "scan-thread", usage); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "gpt-5.6-sol" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const writeUnpricedUsage = async () => { + const path = await writeUsageSession( + codexHome, + "unpriced-worker", + { input_tokens: 100, output_tokens: 10 }, + "scan-thread", + ); + const lines = (await readFile(path, "utf8")).split("\n"); + lines.splice( + 1, + 0, + JSON.stringify({ + type: "turn_context", + payload: { model: "synthetic-unpriced-model" }, + }), + ); + await writeFile(path, lines.join("\n")); + }; + const client = new TestClient( + {}, + { + environment: {}, + prepareRuntime: async () => preparedRuntime(codexHome), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + commands.push(args); + if (args[0] === "get-scan") + return { scan: { id: "scan_example_001" } }; + if (args[0] === "complete-budget-exhausted-scan") + throw new Error("Synthetic canonical output is not ready"); + return mockWorkbench(args, input); + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + async runStreamed( + _input: string, + options: { signal: AbortSignal }, + ) { + turns++; + async function* events(): AsyncGenerator { + if (raised) { + await writePricedUsage({ + input_tokens: 800, + output_tokens: 0, + }); + } else { + await writePricedUsage(knownUsage); + await writeUnpricedUsage(); + } + yield { type: "thread.started", thread_id: "scan-thread" }; + if (raised) { + await increased; + await writeUnpricedUsage(); + const path = join( + codexHome, + "sessions", + "2026", + "07", + "26", + "rollout-scan-thread.jsonl", + ); + await appendFile( + path, + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: knownUsage }, + }, + }) + "\n", + ); + } + await new Promise((resolve) => { + if (options.signal.aborted) resolve(); + else + options.signal.addEventListener( + "abort", + () => resolve(), + { once: true }, + ); + }); + throw new DOMException("aborted", "AbortError"); + } + return { events: events() }; + }, + }), + }), + }, + ); + const keepAlive = setTimeout(() => {}, 10_000); + try { + const failure = await client + .run(repository, { + mode, + maxCostUsd: 0.004, + signal: AbortSignal.timeout(5_000), + postScanPrompt: "No model work after the budget stop.", + ...(raised ? { onBudgetApproaching: () => 0.008 } : {}), + onCost: (cost, limit) => { + costs.push(cost.estimatedUsd); + if (limit === 0.008) releaseIncrease(); + }, + }) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ScanCostLimitExceededError); + expect(failure).toMatchObject({ + maxCostUsd: raised ? 0.008 : 0.004, + cost: { + estimatedUsd: expectedCost.estimatedUsd, + inputTokens: knownUsage.input_tokens, + coverage: "partial", + }, + }); + expect(costs.every((cost) => raised && cost === 0.0032)).toBe(true); + expect(turns).toBe(1); + expect( + commands.some((args) => args[0] === "complete-budget-exhausted-scan"), + ).toBe(mode === "deep"); + if (raised) + expect( + commands + .filter((args) => args[0] === "set-scan-cost-limit") + .map((args) => args.at(-1)), + ).toEqual(["0.008"]); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }, + ); + test("stops and records a scan as soon as its live cost exceeds the limit", async () => { const root = await temporaryDirectory(); const repository = join(root, "repository"); @@ -4930,6 +5107,10 @@ describe("CodexSecurity orchestration", () => { input?: string, ): Promise => { commands.push(args); + if (args[0] === "get-scan") { + // Older workbench readers return a scan without execution attribution. + return { scan: { id: "scan_example_001" } }; + } if (args[0] !== "complete-budget-exhausted-scan") { return mockWorkbench(args, input); } @@ -6341,8 +6522,8 @@ describe("CodexSecurity orchestration", () => { ); expect(prompt).toContain("$codex-security:deep-security-scan"); expect(prompt).not.toContain("record_codex_security_scan_draft"); - expect(prompt).toContain("complete_codex_security_scan"); - expect(prompt).not.toContain("do not finalize or seal them"); + expect(prompt).not.toContain("complete_codex_security_scan"); + expect(prompt).toContain("Leave finalization to the SDK after this turn"); expect(prompt).toContain( 'start_codex_security_deep_scan with {"scanId":"scan_example_001"}', ); @@ -6353,7 +6534,7 @@ describe("CodexSecurity orchestration", () => { await expect( client.run(repository, { mode: "deep", maxCostUsd: 1 }), ).rejects.toThrow("prompt captured"); - expect(prompt).toContain("do not finalize or seal them"); + expect(prompt).toContain("Leave finalization to the SDK after this turn"); expect(prompt).not.toContain("complete_codex_security_scan"); await expect( @@ -7298,6 +7479,211 @@ if ([basename(process.argv[1]), ...process.argv.slice(2)].join(" ") !== "login s expect(scanSignal?.aborted).toBe(false); }); + test.each([ + ["standard", false], + ["deep", false], + ["deep", true], + ] as const)( + "isolates concurrent managed %s sessions at the Codex child boundary (capture=%j)", + async (mode, captureSummary) => { + const clients: TestClient[] = []; + try { + const outcomes = await Promise.allSettled( + ["first", "second"].map(async (name) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + const preload = join(root, "fake-codex.mjs"); + const marker = join(root, "invocation.jsonl"); + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(scanDir, { mode: 0o700 }), + ]); + const model = `fixture-${name}-model`; + const summary = name === "first" ? "none" : "concise"; + const configPath = join(codexHome, "config-preflight.toml"); + let recipeConfig: JsonObject | undefined; + await writeFile( + preload, + [ + 'import { appendFileSync } from "node:fs";', + 'let prompt = ""; for await (const chunk of process.stdin) prompt += chunk;', + `appendFileSync(${JSON.stringify(marker)}, JSON.stringify({args:process.argv, executable:process.execPath, cwd:process.cwd(), home:process.env.CODEX_HOME, key:process.env.CODEX_API_KEY, value:process.env.FIXTURE_SCAN_VALUE, prompt}) + "\\n");`, + `if (process.argv.includes("models")) { console.log(JSON.stringify({models:[{slug:${JSON.stringify(model)},default_reasoning_summary:${JSON.stringify(summary)}}]})); process.exit(0); }`, + `console.log(JSON.stringify({type:"thread.started",thread_id:${JSON.stringify(`fixture-${name}-thread`)}}));`, + 'console.log(JSON.stringify({type:"item.completed",item:{type:"agent_message",text:"scan complete"}}));', + 'console.log(JSON.stringify({type:"turn.completed",usage:null}));', + "process.exit(0);", + ].join("\n"), + ); + const fake = nodeCodex(preload); + const provider = `fixture-${name}-provider`; + const client = new TestClient( + { + codexOverrides: { + model, + model_provider: provider, + model_reasoning_effort: "ultra", + // JavaScript callers can omit the merged default with undefined. + model_reasoning_summary: captureSummary + ? (undefined as unknown as string) + : summary, + service_tier: name === "first" ? "flex" : "fast", + features: { + multi_agent_v2: { max_concurrent_threads_per_session: 4 }, + }, + }, + }, + { + environment: { + OPENAI_API_KEY: `synthetic-${name}-key`, + CODEX_CLI_PATH: fake.command.command, + }, + prepareRuntime: async () => ({ + ...preparedRuntime(codexHome), + configPath, + environment: { + ...fake.environment, + FIXTURE_SCAN_VALUE: name, + }, + }), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args, input) => { + if (args[0] === "register-cli-scan") + recipeConfig = JSON.parse(input!).recipe.config; + return mockWorkbench(args, input); + }, + createCodex: (options: CodexOptions) => { + const codex = new Codex(options); + return { + startThread: (threadOptions: ThreadOptions) => { + const thread = codex.startThread(threadOptions); + return { + get id() { + return thread.id; + }, + runStreamed: async ( + ...args: Parameters + ) => { + if (thread.id === null) { + await copyCompletedScan(root); + if (mode === "deep") { + const coveragePath = join( + scanDir, + "coverage.json", + ); + const coverage = JSON.parse( + await readFile(coveragePath, "utf8"), + ); + coverage.mode = "deep_repository"; + const coverageBytes = JSON.stringify(coverage); + await writeFile(coveragePath, coverageBytes); + const manifestPath = join( + scanDir, + "scan-manifest.json", + ); + const manifest = JSON.parse( + await readFile(manifestPath, "utf8"), + ); + manifest.scan.artifacts.find( + (artifact: { path: string }) => + artifact.path === "coverage.json", + ).sha256 = createHash("sha256") + .update(coverageBytes) + .digest("hex"); + await writeFile( + manifestPath, + JSON.stringify(manifest), + ); + } + } + return thread.runStreamed(...args); + }, + }; + }, + }; + }, + }, + ); + clients.push(client); + const postScanPrompt = "Summarize the completed synthetic scan."; + const result = await client.run(repository, { + mode, + postScanPrompt, + }); + expect(result.threadId).toBe(`fixture-${name}-thread`); + expect(result.turnResult.usage).toBeNull(); + const invocations = (await readFile(marker, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + const lookups = invocations.filter((child) => + child.args.includes("models"), + ); + const children = invocations.filter( + (child) => !child.args.includes("models"), + ); + expect(recipeConfig?.["model_reasoning_summary"]).toBe(summary); + expect(lookups).toHaveLength(captureSummary ? 1 : 0); + for (const lookup of lookups) { + expect(await realpath(lookup.cwd)).toBe(await realpath(scanDir)); + expect(lookup.home).toBe(codexHome); + expect(lookup.key).toBe(`synthetic-${name}-key`); + expect(lookup.value).toBe(name); + expect(lookup.args).toContain(`model=${JSON.stringify(model)}`); + } + const preflight = await readFile(configPath, "utf8"); + expect(parseToml(preflight)["model_reasoning_summary"]).toBe( + summary, + ); + expect(preflight).not.toContain(`synthetic-${name}-key`); + expect(children).toHaveLength(2); + expect(children[1].prompt).toBe(postScanPrompt); + expect(children[1].args).toContain("resume"); + expect(children[1].args).toContain(`fixture-${name}-thread`); + for (const child of children) { + expect(child.executable).toBe( + process.platform === "win32" + ? win32.toNamespacedPath(fake.command.command) + : fake.command.command, + ); + expect(child.home).toBe(codexHome); + expect(child.key).toBe(`synthetic-${name}-key`); + expect(child.value).toBe(name); + expect(child.args).toContain(`model=${JSON.stringify(model)}`); + expect(child.args).toContain( + `model_provider=${JSON.stringify(provider)}`, + ); + expect(child.args).toContain('model_reasoning_effort="ultra"'); + expect(child.args).toContain( + `model_reasoning_summary=${JSON.stringify(name === "first" ? "none" : "concise")}`, + ); + expect(child.args).toContain( + `service_tier=${JSON.stringify(name === "first" ? "flex" : "fast")}`, + ); + expect(child.args).toContain( + "features.multi_agent_v2.max_concurrent_threads_per_session=4", + ); + expect(child.args).toContain( + 'default_permissions="codex_security_scan"', + ); + expect(child.args).toContain('approval_policy="on-request"'); + } + }), + ); + for (const outcome of outcomes) { + if (outcome.status === "rejected") throw outcome.reason; + } + } finally { + await Promise.all(clients.map((client) => client.close())); + } + }, + ); + test("closes a real Codex subprocess cleanly after a streamed terminal failure", async () => { const root = await temporaryDirectory(); const repository = join(root, "repository"); diff --git a/sdk/typescript/tests-ts/build-plugin.test.ts b/sdk/typescript/tests-ts/build-plugin.test.ts index ecd012ee1..db52ed062 100644 --- a/sdk/typescript/tests-ts/build-plugin.test.ts +++ b/sdk/typescript/tests-ts/build-plugin.test.ts @@ -1,19 +1,23 @@ import { execFile } from "node:child_process"; import { chmod, + copyFile, + cp, mkdir, mkdtemp, readFile, readdir, rm, stat, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { afterEach, describe, expect, test } from "bun:test"; +import { transform } from "esbuild"; import { buildBundledPlugin } from "../scripts/build-plugin.mjs"; import { assertGeneratedPluginUntracked } from "../scripts/check-plugin-source.mjs"; @@ -69,8 +73,163 @@ afterEach(async () => { }); describe("bundled plugin build", () => { - test("builds the MCP runtime without invoking an npm launcher", async () => { + test("bundles the native policy proof with only SDK dependencies", async () => { const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const native = join(plugin, "native"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(native, { recursive: true }); + await mkdir(sdk, { recursive: true }); + await symlink( + fileURLToPath(new URL("../node_modules", import.meta.url)), + join(sdk, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + for (const name of ["schemas", "mcp-app/src"]) { + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + await copyFile( + new URL("mcp-app/helpers-main.ts", source), + join(plugin, "mcp-app", "helpers-main.ts"), + ); + for (const name of [ + "binding", + "platform", + "windows-binding", + "windows-flags", + "windows-files", + "proof-policy-windows", + ]) { + const compiled = await transform( + await readFile(new URL(`native/${name}.mts`, source), "utf8"), + { loader: "ts", format: "esm", target: "node20" }, + ); + await writeFile(join(native, `${name}.mjs`), compiled.code); + } + const { nativeTarget } = await import( + pathToFileURL(join(native, "platform.mjs")).href + ); + const binary = process.platform === "win32" ? "windows.node" : "unix.node"; + // The build copies this artifact; this portable test does not load native code. + await writeFixture( + native, + `dist/${nativeTarget}/${binary}`, + "native fixture", + ); + await expect( + stat(join(plugin, "mcp-app", "node_modules")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + await execFileAsync( + "node", + [join(native, "proof-policy-windows.mjs"), "build"], + { + cwd: native, + env: { ...process.env, NODE_PATH: "" }, + }, + ); + const proof = join(native, "dist", nativeTarget, "policy-proof"); + expect( + await readFile( + join(proof, "native", nativeTarget, "windows.node"), + "utf8", + ), + ).toBe("native fixture"); + const helper = join(root, "helpers.cjs"); + await copyFile(join(proof, "helpers.cjs"), helper); + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(sdk, "node_modules"), + ]); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const result = await execFileAsync( + "node", + [ + "--eval", + ` + const assert = require("node:assert/strict"); + const helper = require(process.argv.pop()); + const input = { + scanId: "synthetic-scan", + manifest: { scan: {} }, + findings: { findings: [] }, + coverage: { + completeness: "complete", surfaces: [], explicitExclusions: [], deferred: [], + }, + }; + assert.equal(helper.parseCanonicalScanDraft(input).scanId, input.scanId); + assert.throws(() => helper.parseCanonicalScanDraft({ + ...input, coverage: { ...input.coverage, completeness: "invalid" }, + })); + console.log("Bundled parser accepted valid input and rejected invalid coverage."); + `, + helper, + ], + { cwd: root, env: { ...process.env, NODE_PATH: "" } }, + ); + expect(result.stdout).toBe( + "Bundled parser accepted valid input and rejected invalid coverage.\n", + ); + expect(result.stderr).toBe(""); + }); + + test("builds the MCP runtime with only MCP dependencies and no npm launcher", async () => { + const root = await temporaryDirectory(); + const plugin = join(root, "plugins", "codex-security"); + const mcp = join(plugin, "mcp-app"); + const sdk = join(root, "sdk", "typescript"); + const source = new URL("../../../plugins/codex-security/", import.meta.url); + await mkdir(mcp, { recursive: true }); + await mkdir(sdk, { recursive: true }); + for (const name of [ + "package.json", + "tsconfig.json", + "main.ts", + "artifact-writer-main.ts", + "helpers-main.ts", + "server.ts", + "src", + "scripts", + "templates", + ]) { + await cp(new URL(`mcp-app/${name}`, source), join(mcp, name), { + recursive: true, + }); + } + for (const name of [ + "schemas", + "native/prebuilt", + "plugin-files.json", + "scripts/reserved_artifact_paths.json", + ]) { + await cp(new URL(name, source), join(plugin, name), { recursive: true }); + } + for (const name of await readdir(new URL("native/", source))) { + if (/\.(?:mjs|mts)$/.test(name)) { + await copyFile( + new URL(`native/${name}`, source), + join(plugin, "native", name), + ); + } + } + for (const name of ["src", "package.json", "tsconfig.json"]) { + await cp(new URL(`../${name}`, import.meta.url), join(sdk, name), { + recursive: true, + }); + } + await symlink( + fileURLToPath(new URL("mcp-app/node_modules", source)), + join(mcp, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + await expect(stat(join(sdk, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); const bin = join(root, "bin"); const launcher = process.platform === "win32" ? "npm.cmd" : "npm"; await writeFixture( @@ -83,24 +242,24 @@ describe("bundled plugin build", () => { const destination = join(root, "mcp"); await execFileAsync( "node", - [ - fileURLToPath( - new URL( - "../../../plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs", - import.meta.url, - ), - ), - "--output", - destination, - ], + [join(mcp, "scripts", "build_mcp_app.mjs"), "--output", destination], { env: { ...process.env, + NODE_PATH: "", PATH: [bin, process.env["PATH"]].filter(Boolean).join(delimiter), }, }, ); + await execFileAsync("node", [ + "--eval", + "require('node:fs').unlinkSync(process.argv[1])", + join(mcp, "node_modules"), + ]); + await expect(stat(join(mcp, "node_modules"))).rejects.toMatchObject({ + code: "ENOENT", + }); const contract = JSON.parse( await readFile( new URL( @@ -125,6 +284,61 @@ describe("bundled plugin build", () => { ]); expect(helper.stdout).toBe("[]\n"); expect(helper.stderr).toBe(""); + + const repository = await temporaryDirectory(); + const policy = "Preserve this synthetic inherited security policy."; + await writeFixture( + repository, + "SECURITY.md", + `# Synthetic policy\n${policy}\n`, + ); + const alias = join(await temporaryDirectory(), "plugin link"); + await symlink( + root, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + await mkdir(join(root, "scripts"), { recursive: true }); + await copyFile( + new URL("scripts/launch_codex_security_mcp", source), + join(root, "scripts", "launch_codex_security_mcp"), + ); + const node = ( + await execFileAsync("node", ["--print", "process.execPath"]) + ).stdout.trim(); + for (const pluginPath of [root, alias]) { + const linkedHelper = join(pluginPath, "mcp", "helpers.mjs"); + const list = await execFileAsync( + process.platform === "win32" ? node : "/bin/sh", + [ + ...(process.platform === "win32" + ? [linkedHelper] + : [ + join(pluginPath, "scripts", "launch_codex_security_mcp"), + "--helper", + ]), + "resolve-security-md", + "--repo", + repository, + "--list", + ], + { env: { ...process.env, CODEX_MCP_NODE_PATH: node, NODE_PATH: "" } }, + ); + expect(list.stdout).toBe('["SECURITY.md"]\n'); + expect(list.stderr).toBe(""); + const guidance = await execFileAsync(node, [ + linkedHelper, + "resolve-security-md", + "--repo", + repository, + "--scope", + repository, + "--out", + "-", + ]); + expect(guidance.stdout).toContain(policy); + expect(guidance.stderr).toBe(""); + } }); test("builds from a source snapshot without Git metadata", async () => { diff --git a/sdk/typescript/tests-ts/cli-patch-results.test.ts b/sdk/typescript/tests-ts/cli-patch-results.test.ts index febd49243..1bf647627 100644 --- a/sdk/typescript/tests-ts/cli-patch-results.test.ts +++ b/sdk/typescript/tests-ts/cli-patch-results.test.ts @@ -117,7 +117,7 @@ lines.on("line", (line) => { }); test.each([false, true])( - "fails a no-op with full output %s and preserves local changes", + "fails a no-op with full output %j and preserves local changes", async (fullOutput) => { const fixture = await repositoryFixture(); await writeFile(join(fixture.directory, "app.ts"), "staged change\n"); @@ -178,7 +178,7 @@ lines.on("line", (line) => { }); test.each([false, true])( - "checks patch changes outside a Git repository: %s", + "checks patch changes outside a Git repository: %j", async (apply) => { const fixture = await repositoryFixture(); await rm(join(fixture.directory, ".git"), { diff --git a/sdk/typescript/tests-ts/cli-scan-logs.test.ts b/sdk/typescript/tests-ts/cli-scan-logs.test.ts new file mode 100644 index 000000000..42fb6ea49 --- /dev/null +++ b/sdk/typescript/tests-ts/cli-scan-logs.test.ts @@ -0,0 +1,470 @@ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Writable } from "node:stream"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; +import { describe, expect, test } from "bun:test"; +import { Cli, Formatter, z } from "incur"; +import { main } from "../src/cli.js"; +import { scanLogsJson } from "../src/cli-scan-logs-json.js"; +import { readSavedScanLogs } from "../src/scan-logs.js"; +import { VERSION } from "../src/version.js"; +import { capture, dependencies } from "./cli-fixtures.js"; + +async function fixture(attributedOwner = false) { + const state = await realpath(await mkdtemp(join(tmpdir(), "saved-logs-"))); + const home = join(state, "codex-home"); + await mkdir(join(home, "sessions"), { recursive: true }); + const events: Record[] = [ + { type: "session_meta", payload: { id: "thread-1" } }, + { + type: "event_msg", + payload: { + message: 'full values: \" \\ \n \u0000 😀', + nested: [null, false, 2], + }, + }, + ]; + const timestamp = "2026-08-11T12:01:00.000Z"; + if (attributedOwner) { + events.splice(1, 0, { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }); + Object.assign(events.at(-1)!, { timestamp }); + } + await writeFile( + join(home, "sessions", "rollout.jsonl"), + events.map((event) => JSON.stringify(event)).join("\n"), + ); + const originalHome = join(state, "original-home"); + const scanDir = join(state, "scan"); + const settingsDirectory = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settingsDirectory, { recursive: true }); + await mkdir(join(originalHome, "sessions"), { recursive: true }); + let ownerEvents = events; + if (attributedOwner) { + const repeated = { + type: "event_msg", + timestamp, + payload: { message: "repeated scan occurrence" }, + }; + ownerEvents = [ + ...events, + repeated, + repeated, + { + type: "event_msg", + timestamp, + payload: { message: "recorded non-usage suffix" }, + }, + ]; + await writeFile( + join(home, "sessions", "rollout.jsonl"), + [ + ...events, + { type: "turn_context", timestamp, payload: { turn_id: "other-turn" } }, + { + type: "event_msg", + timestamp, + payload: { message: "unrelated first-copy suffix" }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n"), + ); + await writeFile( + join(originalHome, "sessions", "owner.jsonl"), + ownerEvents.map((event) => JSON.stringify(event)).join("\n"), + ); + } + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { + codexHome: originalHome, + codexPath: join(originalHome, "codex"), + }, + }), + ); + await writeFile( + join(home, "sessions", "worker.jsonl"), + JSON.stringify({ type: "session_meta", payload: { id: "worker" } }) + "\n", + ); + await writeFile( + join(originalHome, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + { type: "event_msg", payload: { message: "recorded worker suffix" } }, + ] + .map((event) => JSON.stringify(event) + "\n") + .join(""), + ); + const scan = { + scanId: "scan-1", + continuationThreadId: "thread-1", + mode: "deep", + scanDir, + executionThreadIds: ["worker"], + ...(attributedOwner + ? { + executionAttribution: { + formatVersion: 1 as const, + executionThreadIds: ["worker"], + owner: { + threadId: "thread-1", + turnId: "scan-turn", + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + } + : {}), + }; + const logs = await readSavedScanLogs(scan, [home, originalHome]); + const deps = dependencies({ + environment: { CODEX_SECURITY_STATE_DIR: state }, + onWorkbench: () => ({ scan }), + }); + deps.createSecurity = () => { + throw new Error("Reading logs must not start Codex"); + }; + return { state, logs, deps, ownerEvents }; +} + +async function referenceOutput(args: string[], logs: unknown) { + let text = ""; + await Cli.create("codex-security", { version: VERSION, update: false }) + .command( + Cli.create("scans").command("logs", { + args: z.object({ scanId: z.string() }), + run: () => logs, + }), + ) + .serve(["scans", "logs", "scan-1", ...args], { + stdout: (value) => { + text += value; + }, + }); + return text; +} + +function withoutDuration(text: string) { + const value = JSON.parse(text); + value.meta.duration = "duration"; + return value; +} + +describe("saved logs JSON output", () => { + test("selects the recorded owner suffix after scan attribution through the saved logs command", async () => { + const f = await fixture(true); + try { + const stdout = capture(); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + stdout.stream, + capture().stream, + f.deps, + ), + ).toBe(0); + const result = JSON.parse(stdout.text()); + expect( + result.sessions.map(({ threadId }: { threadId: string }) => threadId), + ).toEqual(["worker", "thread-1"]); + expect( + result.events + .filter( + ({ threadId }: { threadId: string }) => threadId === "thread-1", + ) + .map(({ event }: { event: unknown }) => event), + ).toEqual(f.ownerEvents); + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + + test("loads the same-thread recorded worker suffix through the saved logs command", async () => { + const f = await fixture(); + try { + const stdout = capture(); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + stdout.stream, + capture().stream, + f.deps, + ), + ).toBe(0); + expect( + JSON.parse(stdout.text()).sessions.map( + ({ threadId }: { threadId: string }) => threadId, + ), + ).toEqual(["thread-1", "worker"]); + expect(JSON.parse(stdout.text()).events).toContainEqual({ + threadId: "worker", + event: { + type: "event_msg", + payload: { message: "recorded worker suffix" }, + }, + }); + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + + test("preserves the stale installed-skills CTA after saved logs", async () => { + const f = await fixture(); + const previousDataHome = process.env["XDG_DATA_HOME"]; + try { + const dataHome = join(f.state, "data"); + const skillPath = join(f.state, "skills", "codex-security-scans"); + await mkdir(join(dataHome, "incur"), { recursive: true }); + await mkdir(skillPath, { recursive: true }); + await writeFile( + join(skillPath, "SKILL.md"), + "Previously installed skill.", + ); + await writeFile( + join(dataHome, "incur", "codex-security.json"), + JSON.stringify({ + hash: "previous-command-hash", + skills: ["codex-security-scans"], + paths: [skillPath], + }), + ); + process.env["XDG_DATA_HOME"] = dataHome; + for (const args of [ + ["--json"], + ["--format", "json"], + ["--format=json"], + ]) { + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + f.deps, + ), + ).toBe(0); + const expected = await referenceOutput(["--json"], f.logs); + expect(Object.keys(JSON.parse(expected))).toEqual([ + "scanId", + "threadId", + "sessions", + "events", + "cta", + ]); + expect(stdout.text()).toBe(expected); + expect(stderr.text()).toBe(""); + } + } finally { + if (previousDataHome === undefined) delete process.env["XDG_DATA_HOME"]; + else process.env["XDG_DATA_HOME"] = previousDataHome; + await rm(f.state, { recursive: true, force: true }); + } + }); + + test.each( + [ + [], + ["--json"], + ["--format", "json"], + ["--format=json"], + ["--format", "toon"], + ["--format", "jsonl"], + ["--format", "yaml"], + ["--format", "md"], + ["--json", "--format", "toon"], + ["--format", "toon", "--json"], + ["--json", "--filter-output", "scanId"], + ["--json", "--filter-output", "events[0,1]"], + ["--json", "--filter-output", "missing"], + ["--json", "--filter-output", "scanId,threadId"], + ["--json", "--token-count"], + ["--json", "--token-limit", "4"], + ["--json", "--token-offset", "1", "--token-limit", "4"], + ["--json", "--token-offset", "999999"], + ["--json", "--token-limit", "999999"], + ["--json", "--full-output"], + ["--json", "--full-output", "--filter-output", "scanId"], + ["--json", "--full-output", "--token-limit", "4"], + ["--json", "--full-output", "--token-count"], + ].map((args) => [args]), + )("preserves Incur output for %j", async (args) => { + const f = await fixture(); + try { + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + f.deps, + ), + ).toBe(0); + const expected = await referenceOutput( + args.flatMap((arg) => + arg === "--format=json" ? ["--format", "json"] : [arg], + ), + f.logs, + ); + if (args.includes("--full-output") && !args.includes("--token-count")) { + expect(withoutDuration(stdout.text())).toEqual( + withoutDuration(expected), + ); + } else { + expect(stdout.text()).toBe(expected); + } + expect(stderr.text()).toBe(""); + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + + test.each( + [ + ["--format", "invalid"], + ["--format=invalid"], + ["--json", "--token-limit", "invalid"], + ["--json", "--token-offset", "invalid"], + ["--json", "--filter-output"], + ].map((args) => [args]), + )("preserves invalid-option failure for %j", async (args) => { + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scans", "logs", "scan-1", ...args], + stdout.stream, + stderr.stream, + dependencies(), + ), + ).toBe(2); + expect(stdout.text()).toBe(""); + expect(stderr.text()).not.toBe(""); + }); + + test("preserves JSON conversion and empty arrays", async () => { + const f = await fixture(); + try { + f.logs.events.push({ + threadId: "thread-1", + event: { + bigint: 42n, + missing: undefined, + values: [NaN, Infinity, -0], + }, + }); + for (const logs of [f.logs, { ...f.logs, sessions: [], events: [] }]) { + const chunks = []; + for await (const chunk of scanLogsJson(logs)) + chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe( + `${Formatter.format(logs, "json")}\n`, + ); + } + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + + test("waits for output backpressure and leaves the stream open", async () => { + const f = await fixture(); + try { + const chunks: Buffer[] = []; + const output = new Writable({ + highWaterMark: 1, + write(chunk, _encoding, callback) { + setImmediate(() => { + chunks.push(Buffer.from(chunk)); + callback(); + }); + }, + }); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + output, + capture().stream, + f.deps, + ), + ).toBe(0); + expect(Buffer.concat(chunks).toString()).toBe( + `${Formatter.format(f.logs, "json")}\n`, + ); + expect(output.writableEnded).toBe(false); + expect(output.listenerCount("error")).toBe(0); + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + + test("reports a failed output write", async () => { + const f = await fixture(); + try { + const stderr = capture(); + const output = new Writable({ + write(_chunk, _encoding, callback) { + callback(new Error("synthetic write failure")); + }, + }); + expect( + await main( + ["scans", "logs", "scan-1", "--json"], + output, + stderr.stream, + f.deps, + ), + ).toBe(2); + expect(stderr.text()).toContain("synthetic write failure"); + } finally { + await rm(f.state, { recursive: true, force: true }); + } + }); + + test("writes a complete document larger than Node's maximum string", async () => { + // Keep the bundle beneath the SDK so external dependencies resolve normally. + const bundle = await mkdtemp( + join(import.meta.dir, "..", ".logs-boundary-"), + ); + const state = await mkdtemp(join(tmpdir(), "large-saved-logs-")); + try { + const built = await Bun.build({ + entrypoints: [join(import.meta.dir, "support", "cli-logs-large.mts")], + outdir: bundle, + target: "node", + packages: "external", + }); + expect(built.success).toBe(true); + const result = await promisify(execFile)( + "node", + [ + "--input-type=module", + "--eval", + `await import(${JSON.stringify(pathToFileURL(built.outputs[0]!.path).href)})`, + "synthetic-launcher", + state, + ], + { encoding: "utf8" }, + ); + const proof = JSON.parse(result.stdout); + expect(proof.exitCode).toBe(0); + expect(proof.bytes).toBeGreaterThan(proof.maximumStringLength); + expect(proof.bytes).toBe(proof.expectedBytes); + expect(proof.sha256).toBe(proof.expectedSha256); + expect(result.stderr).toBe(""); + } finally { + await rm(bundle, { recursive: true, force: true }); + await rm(state, { recursive: true, force: true }); + } + }); +}); diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index 077eb4983..9c72c9ac1 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -1,10 +1,13 @@ import { spawnSync } from "node:child_process"; import { appendFile, + cp, mkdir, mkdtemp, realpath, + readFile, rm, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -15,9 +18,14 @@ import { estimateScanCost, ScanCostTracker, type ScanSessionEvent, + type ScanCost, } from "../src/cost.js"; import type { ScanActivity } from "../src/scan-activity.js"; -import { formatTokenUsage, tokenUsage } from "../src/cost-model.js"; +import { + estimateScanCostLowerBound, + formatTokenUsage, + tokenUsage, +} from "../src/cost-model.js"; import { readScanLogs } from "../src/scan-logs.js"; import { sessionParentThreadId } from "../src/scan-sessions.js"; import type { ScanProgress } from "../src/worker-progress.js"; @@ -2259,3 +2267,479 @@ describe("live scan cost tracking", () => { }, ); }); + +describe("recorded Deep worker homes", () => { + test("only enforces a priced subtotal from a valid attributed usage partition", () => { + const known = { + model: "gpt-5.6-sol", + input_tokens: 1_000, + output_tokens: 0, + }; + const unknown = { model: null, input_tokens: 100, output_tokens: 0 }; + const usage = { + input_tokens: 1_100, + output_tokens: 0, + modelUsage: [known, unknown], + }; + expect(estimateScanCostLowerBound("gpt-5.6-sol", usage)?.estimatedUsd).toBe( + 0.004, + ); + expect(estimateScanCost("gpt-5.6-sol", usage)).toBeNull(); + for (const invalid of [ + { ...usage, input_tokens: 999 }, + { ...usage, modelUsage: [known, known, unknown] }, + { ...usage, modelUsage: [known, { ...unknown, input_tokens: -1 }] }, + { ...usage, modelUsage: [{ ...known, model: null }, unknown] }, + ]) + expect(estimateScanCostLowerBound("gpt-5.6-sol", invalid)).toBeNull(); + }); + + test.each([null, "synthetic-unpriced-model"])( + "reports an internal priced lower bound with model %p without inventing a total", + async (unknownModel) => { + const home = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const known = await writeSession(home, "owner", {}); + await appendFile( + known, + JSON.stringify({ + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: "owner", + turn_id: "turn", + response_id: "known-response", + model: "gpt-5.6-sol", + usage: { input_tokens: 1_000, output_tokens: 0 }, + }, + }) + "\n", + ); + const unknown = await writeSession(home, "worker", {}); + await appendFile( + unknown, + JSON.stringify({ + type: "turn_context", + timestamp: at, + payload: { + turn_id: "worker-turn", + ...(unknownModel === null ? {} : { model: unknownModel }), + }, + }) + + "\n" + + JSON.stringify({ + type: "event_msg", + timestamp: at, + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + }) + + "\n", + ); + const lowerBounds: Readonly[] = []; + const publicCosts: Readonly[] = []; + const options = { + codexHome: home, + model: "gpt-5.6-sol", + maxCostUsd: 0.003, + onCost: (cost: Readonly) => publicCosts.push(cost), + onCostLowerBound: (cost: Readonly) => lowerBounds.push(cost), + }; + const tracker = new ScanCostTracker(options); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { threadId: "owner", turnId: "turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + tracker.start("owner"); + try { + const snapshot = await tracker.refresh(); + expect(tokenUsage(snapshot.usage)?.input_tokens).toBe(1_100); + expect(snapshot.cost).toBeNull(); + expect(publicCosts).toEqual([]); + expect(lowerBounds).toHaveLength(1); + expect(lowerBounds[0]).toMatchObject({ + inputTokens: 1_000, + estimatedUsd: 0.004, + coverage: "partial", + }); + expect(lowerBounds[0]!.estimatedUsd).toBeGreaterThan( + options.maxCostUsd, + ); + await tracker.refresh(); + expect(lowerBounds).toHaveLength(1); + } finally { + await tracker.stop(); + } + }, + ); + + test.each(["identical", "prefix-first", "prefix-last"] as const)( + "forwards each event occurrence once from copied logs: %s", + async (copy) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const settings = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: recordedHome }, + }), + ); + await mkdir(join(home, "sessions")); + await mkdir(join(recordedHome, "sessions")); + const first = join(home, "sessions", "worker.jsonl"); + const second = join(recordedHome, "sessions", "worker-copy.jsonl"); + const repeated = { + timestamp: "2026-09-01T00:00:02Z", + type: "event_msg", + payload: { type: "agent_message", message: "Reviewing source." }, + }; + const expected = [ + { + timestamp: "2026-09-01T00:00:00Z", + type: "session_meta", + payload: { id: "worker", model: "gpt-5.6-sol" }, + }, + repeated, + repeated, + { + timestamp: "2026-09-01T00:00:03Z", + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: "response", + model: "gpt-5.6-sol", + usage: { input_tokens: 100, output_tokens: 0 }, + }, + }, + ]; + const contents = expected.map((event) => JSON.stringify(event) + "\n"); + await writeFile( + first, + contents.slice(0, copy === "prefix-first" ? 2 : 4).join(""), + ); + await writeFile( + second, + contents.slice(0, copy === "prefix-last" ? 2 : 4).join(""), + ); + const events: ScanSessionEvent[] = []; + const options = { + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event: ScanSessionEvent) => events.push(event), + }; + const tracker = new ScanCostTracker(options); + tracker.start("worker"); + try { + expect((await tracker.refresh()).cost?.inputTokens).toBe(100); + expect(events.map((event) => event.event)).toEqual(expected); + await tracker.refresh(); + expect(events).toHaveLength(expected.length); + // Both logs catch up, then a genuine repeated occurrence is copied later. + await writeFile(first, contents.join("")); + await writeFile(second, contents.join("")); + await appendFile(first, JSON.stringify(repeated) + "\n"); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + await appendFile(second, JSON.stringify(repeated) + "\n"); + await tracker.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + // Re-reading after the owner interval becomes available filters early + // events, but must not renumber the surviving source occurrences. + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { + threadId: "worker", + turnId: "turn", + startedAt: "2026-09-01T00:00:03Z", + }, + startedAt: "2026-09-01T00:00:03Z", + completedAt: "2026-09-01T00:00:04Z", + })); + await tracker.refresh(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + events.length = 0; + const reconstructed = new ScanCostTracker(options); + reconstructed.start("worker"); + await reconstructed.stop(); + expect(events.map((event) => event.event)).toEqual([ + ...expected, + repeated, + ]); + } finally { + await tracker.stop(); + } + }, + ); + + test("keeps resumed worker usage and current parent usage isolated per scan", async () => { + const currentHome = await codexHome(); + const firstHome = await codexHome(); + const secondHome = await codexHome(); + const at = "2026-09-01T00:00:02Z"; + const trackers: ScanCostTracker[] = []; + const fixture = async (home: string, id: string, count: number) => { + const path = await writeSession(home, id, {}); + await appendFile( + path, + [ + { + type: "turn_context", + timestamp: at, + payload: { turn_id: "scan-turn", model: "gpt-5.6-sol" }, + }, + { + type: "token_usage_record", + timestamp: at, + payload: { + thread_id: id, + turn_id: "scan-turn", + response_id: `${id}-response`, + model: "gpt-5.6-sol", + usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", + ); + return path; + }; + try { + const cases = [ + { id: "one", home: firstHome, parent: 10, discovery: 20, reducer: 30 }, + { id: "two", home: secondHome, parent: 11, discovery: 21, reducer: 31 }, + ]; + for (const row of cases) { + const scanDirectory = join(currentHome, "scans", row.id); + const settingsDirectory = join( + scanDirectory, + "artifacts", + "deep_discovery", + ); + await mkdir(settingsDirectory, { recursive: true }); + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: row.home }, + }), + ); + await fixture(currentHome, `${row.id}-parent`, row.parent); + const discovery = await fixture( + row.home, + `${row.id}-discovery`, + row.discovery, + ); + await fixture(row.home, `${row.id}-reducer`, row.reducer); + await fixture(row.home, `${row.id}-unrelated`, 10_000); + // Repeated receipt identity after reconnect must remain one charge. + const duplicate = (await readFile(discovery, "utf8")) + .trim() + .split("\n") + .at(-1)!; + await appendFile(discovery, duplicate + "\n"); + const attribution = { + formatVersion: 1 as const, + executionThreadIds: [`${row.id}-discovery`, `${row.id}-reducer`], + owner: { + threadId: `${row.id}-parent`, + turnId: "scan-turn", + startedAt: at, + }, + startedAt: at, + completedAt: null, + }; + const tracker = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => attribution); + tracker.start(`${row.id}-parent`); + trackers.push(tracker); + } + const initial = await Promise.all( + trackers.map((tracker) => tracker.refresh()), + ); + expect( + initial.map((snapshot) => tokenUsage(snapshot.usage)?.input_tokens), + ).toEqual([60, 63]); + expect(initial.map((snapshot) => snapshot.cost?.inputTokens)).toEqual([ + 60, 63, + ]); + const firstDirectory = join(currentHome, "scans", "one"); + const rebuilt = new ScanCostTracker({ + codexHome: currentHome, + scanDirectory: firstDirectory, + model: "gpt-5.6-sol", + }); + rebuilt.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: [ + "one-discovery", + "one-reducer", + "one-missing-attempt", + ], + owner: { threadId: "one-parent", turnId: "scan-turn", startedAt: at }, + startedAt: at, + completedAt: null, + })); + rebuilt.start("one-parent"); + trackers.push(rebuilt); + expect((await rebuilt.refresh()).usage).toMatchObject({ + input_tokens: 60, + coverage: "partial", + }); + await fixture(firstHome, "one-missing-attempt", 7); + expect((await rebuilt.refresh()).cost?.inputTokens).toBe(67); + expect((await trackers[1]!.refresh()).cost?.inputTokens).toBe(63); + } finally { + await Promise.all(trackers.map((tracker) => tracker.stop())); + } + }); + + test("reads a recorded directory alias only once", async () => { + const home = await codexHome(); + const alias = join(await codexHome(), "recorded-home"); + await symlink( + home, + alias, + process.platform === "win32" ? "junction" : "dir", + ); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: alias } }), + ); + await writeSession(home, "worker", { input_tokens: 100, output_tokens: 0 }); + const events: ScanSessionEvent[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + onSessionEvent: (event) => events.push(event), + }); + tracker.start("worker"); + try { + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + expect(events).toHaveLength(2); + } finally { + await tracker.stop(); + } + }); + + test.each([ + ["identical", false], + ["identical", true], + ["prefix-first", false], + ["prefix-first", true], + ["prefix-last", false], + ["prefix-last", true], + ] as const)( + "prices copied response records (%s, attribution: %j)", + async (copy, attributed) => { + const home = await codexHome(); + const recordedHome = await codexHome(); + const scanDirectory = join(home, "scan"); + const directory = join(scanDirectory, "artifacts", "deep_discovery"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: recordedHome } }), + ); + const path = await writeSession(home, "worker", {}); + for (const [id, model, input, output] of [ + ["response-one", "gpt-5.6-sol", 100, 10], + ["response-two", "gpt-6-astra", 50, 5], + ] as const) { + await appendFile( + path, + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-09-01T00:00:02Z", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: { input_tokens: input, output_tokens: output }, + }, + }) + "\n", + ); + } + await mkdir(join(recordedHome, "sessions")); + const copiedPath = join(recordedHome, "sessions", "copied-worker.jsonl"); + await cp(path, copiedPath); + const prefix = + (await readFile(path, "utf8")) + .trimEnd() + .split("\n") + .slice(0, -1) + .join("\n") + "\n"; + if (copy === "prefix-first") await writeFile(path, prefix); + if (copy === "prefix-last") await writeFile(copiedPath, prefix); + const tracker = new ScanCostTracker({ + codexHome: home, + scanDirectory, + model: "gpt-5.6-sol", + }); + if (attributed) + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + executionThreadIds: ["worker"], + owner: { + threadId: null, + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + tracker.start("worker"); + try { + const snapshot = await tracker.stop(); + expect(snapshot.usage).toMatchObject({ + input_tokens: 150, + output_tokens: 15, + total_tokens: 165, + }); + expect( + Object.fromEntries( + snapshot.cost!.modelCosts!.map((part) => [ + part.model, + [part.inputTokens, part.outputTokens], + ]), + ), + ).toEqual({ + "gpt-5.6-sol": [100, 10], + "gpt-6-astra": [50, 5], + }); + } finally { + await tracker.stop(); + } + }, + ); +}); diff --git a/sdk/typescript/tests-ts/deep-finalization.test.ts b/sdk/typescript/tests-ts/deep-finalization.test.ts new file mode 100644 index 000000000..2c83534de --- /dev/null +++ b/sdk/typescript/tests-ts/deep-finalization.test.ts @@ -0,0 +1,1150 @@ +import { execFileSync, spawn } from "node:child_process"; +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import type { ThreadEvent } from "@openai/codex-sdk"; +import { + ScanCostLimitExceededError, + ScanInterruptedError, +} from "../src/errors.js"; +import { + prepareScanArtifactRestorer, + runWorkbench, + type WorkbenchCommandOptions, +} from "../src/runtime.js"; +import { resumeSelectedDeepScan } from "../src/deep-scan-finalization.js"; +import { TestClient } from "./support/api-client.js"; +import { + completedEvents, + createApiTestFixtures, + preparedRuntime, +} from "./support/api-events.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const { temporaryDirectory, cleanup } = createApiTestFixtures(); +afterEach(cleanup); +const threadId = "1af317a1-c9ed-4c73-b428-cb0d160cf8e8"; +const followUp = "Explain the selected finding."; + +for (const boundary of ["registration", "stream-start"] as const) { + test(`SDK cancels registered Deep Scan before first thread event: ${boundary}`, async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const codexHome = join(root, "codex-home"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome), + ]); + await writeFile(join(repository, "source.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: join(root, "state"), + }; + const cancellation = new AbortController(); + const reason = new Error( + "Synthetic cancellation before first thread event", + ); + const commands: string[][] = []; + let scanId = ""; + let savedOptions: WorkbenchCommandOptions; + let startedTurns = 0; + const client = new TestClient( + {}, + { + environment, + prepareRuntime: async () => ({ + ...preparedRuntime(codexHome), + environment, + }), + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + savedOptions = options; + commands.push([...args]); + const result = await runWorkbench(options, args, input); + if (args[0] === "register-cli-scan") { + scanId = result["scanId"] as string; + if (boundary === "registration") cancellation.abort(reason); + } + return result; + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + runStreamed: async () => { + startedTurns++; + cancellation.abort(reason); + throw reason; + }, + }), + }), + }, + ); + try { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }) + .catch((error: unknown) => error); + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe(reason); + const stopped = await runWorkbench( + { ...savedOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + }); + expect(commands.filter((args) => args[0] === "cancel-scan")).toEqual([ + ["cancel-scan", "--scan-id", scanId], + ]); + expect(commands.some((args) => args[0] === "fail-scan")).toBe(false); + expect(commands.some((args) => args[0] === "set-scan-thread")).toBe( + false, + ); + expect(startedTurns).toBe(boundary === "registration" ? 0 : 1); + } finally { + await client.close(); + } + }, 30_000); +} + +const outcomes = [ + "failed", + "completed", + "completed-owner-suffix", + "completed-owner-explicit", + "restart", + "canceled-before-publication", + "canceled-during-publication", + "canceled-during-resumed-publication", + "published-before-cancellation", + "closed-during-publication", + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + "closed-during-resumed-publication", + "lost-completion-response", + "lost-completion-response-followup-canceled", + "completion-before-commit-fails", + "followup-canceled", +] as const; +type BudgetCompletionFault = "lost" | "before-commit" | "lost-and-canceled"; +const cases: { + outcome: (typeof outcomes)[number]; + budgetCompletionFault?: BudgetCompletionFault; + initialResumeUsage?: boolean; + unpricedUsage?: boolean; + cancellationFault?: "status-read" | "deep-state-read" | "cancel-response"; +}[] = [ + ...outcomes.map((outcome) => ({ outcome })), + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).flatMap((outcome) => [ + { outcome, unpricedUsage: true }, + { outcome, unpricedUsage: true, budgetCompletionFault: "lost" as const }, + ]), + ...( + [ + "budget-during-publication", + "budget-after-deep-finish", + "budget-during-resumed-publication", + ] as const + ).map((outcome) => ({ outcome, budgetCompletionFault: "lost" as const })), + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "before-commit", + }, + { + outcome: "budget-after-deep-finish", + budgetCompletionFault: "lost-and-canceled", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "status-read", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "cancel-response", + }, + { + outcome: "canceled-during-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-before-publication", + cancellationFault: "deep-state-read", + }, + { + outcome: "canceled-during-resumed-publication", + initialResumeUsage: true, + }, +]; +for (const { + outcome, + budgetCompletionFault, + cancellationFault, + initialResumeUsage, + unpricedUsage, +} of cases) { + const resumedStop = outcome.includes("-resumed-"); + const restart = outcome === "restart" || resumedStop; + const closed = outcome.startsWith("closed-"); + const budgeted = outcome.startsWith("budget-"); + const canceledFollowUp = outcome.endsWith("followup-canceled"); + const loseCompletionResponse = outcome.startsWith("lost-completion-response"); + const name = + outcome === "followup-canceled" + ? "SDK preserves a selected aggregate when its follow-up is canceled" + : `SDK handles selected aggregate: ${outcome}${budgetCompletionFault ? ` (budget completion ${budgetCompletionFault})` : ""}${cancellationFault ? ` (cancellation ${cancellationFault})` : ""}${initialResumeUsage ? " (initial resume cost)" : ""}${unpricedUsage ? " (unpriced remainder)" : ""}`; + const runCase = async () => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const scanDir = join(root, "scan"); + const codexHome = join(root, "codex-home"); + const stateDir = join(root, "state"); + await Promise.all([ + mkdir(repository), + mkdir(scanDir, { mode: 0o700 }), + mkdir(codexHome), + ]); + await writeFile(join(repository, "extract.py"), "# Synthetic source\n"); + const environment = { + ...process.env, + CODEX_HOME: codexHome, + CODEX_SECURITY_STATE_DIR: stateDir, + }; + const cancellation = new AbortController(); + const parentError = new Error( + "Parent turn ended before its final completion tool call", + ); + let scanId = ""; + let workbenchOptions: WorkbenchCommandOptions; + let publicationFails = restart; + let completionReceiptLost = loseCompletionResponse; + let budgetReceiptLost = + budgetCompletionFault === "lost" || + budgetCompletionFault === "lost-and-canceled"; + let budgetTriggered = false; + let cancellationReadLost = false; + let lostCancellationDeepState: unknown; + const ownerSuffix = + outcome === "completed-owner-suffix" || + outcome === "completed-owner-explicit"; + let nativeOwnerStatus: unknown; + let originalFinalizationInput: unknown; + let selectedPath = ""; + let selectedBytes: Buffer; + let originalResumeSignal: AbortSignal | undefined; + let acceptedReport = ""; + let completedArtifacts: Buffer[] = []; + const modelInputs: string[] = []; + const commands: string[] = []; + const reportedCosts: number[] = []; + const usagePath = join( + codexHome, + "sessions", + "2026", + "01", + "01", + `rollout-${threadId}.jsonl`, + ); + const recordBudgetUsage = ( + inputTokens = 1_250, + outputTokens = 30, + timestamp = new Date().toISOString(), + ) => + appendFile( + usagePath, + JSON.stringify({ + timestamp, + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "gpt-5.6-sol", + }, + }) + + "\n" + + JSON.stringify({ + timestamp, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + cached_input_tokens: 200, + output_tokens: outputTokens, + }, + }, + }, + }) + + "\n" + + (unpricedUsage + ? [ + JSON.stringify({ + timestamp, + type: "turn_context", + payload: { + turn_id: "synthetic-scan-turn", + model: "synthetic-unpriced-model", + }, + }), + JSON.stringify({ + timestamp, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 1_350, + cached_input_tokens: 200, + output_tokens: outputTokens, + }, + }, + }, + }), + "", + ].join("\n") + : ""), + ); + let closePromise: Promise | undefined; + const makeClient = () => + new TestClient( + {}, + { + environment, + prepareScanArtifactRestorer, + prepareRuntime: async () => { + const runtime = preparedRuntime(codexHome); + const manifest = JSON.parse( + await readFile( + join(PLUGIN_ROOT, ".codex-plugin/plugin.json"), + "utf8", + ), + ); + return { + ...runtime, + environment, + persistentCredentialHome: true, + plugin: { ...runtime.plugin, version: manifest.version }, + }; + }, + resolvePluginPython: async () => "python3", + prepareOutputDir: async () => scanDir, + runWorkbench: async (options, args, input) => { + workbenchOptions = options; + if (args[0] === "get-cli-scan-resume") + originalResumeSignal = options.signal; + commands.push(args[0]!); + if ( + args[0] === "get-scan" && + cancellation.signal.aborted && + cancellationFault === "status-read" + ) { + throw new Error("Synthetic lost cancellation status response"); + } + if (args[0] === "write-scan-draft" && publicationFails) { + publicationFails = false; + throw new Error("Synthetic publication write failure"); + } + if ( + args[0] === "complete-scan" && + outcome === "completion-before-commit-fails" + ) + throw new Error("Synthetic completion failure before commit"); + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetCompletionFault === "before-commit" + ) + throw new Error( + "Synthetic budget completion failure before commit", + ); + const result = await runWorkbench(options, args, input); + if ( + args[0] === "get-deep-scan" && + cancellation.signal.aborted && + cancellationFault === "deep-state-read" && + !cancellationReadLost + ) { + cancellationReadLost = true; + lostCancellationDeepState = result["deepScan"]; + throw new Error( + "Synthetic lost cancellation Deep state response", + ); + } + if ( + args[0] === "cancel-scan" && + cancellationFault === "cancel-response" + ) { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + throw new Error("Synthetic lost cancellation response"); + } + if ( + args[0] === "complete-budget-exhausted-scan" && + budgetReceiptLost + ) { + budgetReceiptLost = false; + if (budgetCompletionFault === "lost-and-canceled") { + completedArtifacts = await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ); + cancellation.abort( + "Synthetic cancellation after budget completion", + ); + } + throw Object.assign( + new Error("Synthetic lost budget completion response"), + { code: "ETIMEDOUT" }, + ); + } + if (args[0] === "complete-scan" && completionReceiptLost) { + completionReceiptLost = false; + throw new Error("Synthetic lost completion response"); + } + if ( + !budgetTriggered && + ((args[0] === "write-scan-draft" && + (outcome === "budget-during-publication" || + outcome === "budget-during-resumed-publication")) || + (args[0] === "finish-deep-scan" && + outcome === "budget-after-deep-finish")) + ) { + budgetTriggered = true; + await recordBudgetUsage(); + await new Promise((resolve) => { + if (options.signal?.aborted) resolve(); + else + options.signal!.addEventListener("abort", () => resolve(), { + once: true, + }); + }); + } + if (args[0] === "write-scan-draft" && closed) { + closePromise = client.close(); + } + if ( + args[0] === "write-scan-draft" && + (outcome === "canceled-during-publication" || + outcome === "canceled-during-resumed-publication") + ) { + if (initialResumeUsage) { + expect(originalResumeSignal?.reason).toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect(options.signal?.aborted).toBe(false); + } + cancellation.abort("Synthetic user cancellation"); + } + if ( + args[0] === "complete-scan" && + outcome === "published-before-cancellation" + ) { + cancellation.abort( + "Synthetic user cancellation after completion", + ); + } + if (args[0] === "register-cli-scan") + scanId = result["scanId"] as string; + return result; + }, + createCodex: () => { + const thread = { + id: threadId, + async runStreamed(input: string) { + modelInputs.push(input); + if (input === followUp) { + if (canceledFollowUp) { + const reportPath = join(scanDir, "report.md"); + acceptedReport = await readFile(reportPath, "utf8"); + expect(acceptedReport).toContain( + "Validate the resolved destination", + ); + await writeFile( + reportPath, + "Incomplete follow-up report.\n", + ); + cancellation.abort( + "Synthetic cancellation during follow-up", + ); + } + return { events: completedEvents(threadId) }; + } + expect(modelInputs.length).toBe(1); + async function* events(): AsyncGenerator { + yield { type: "thread.started", thread_id: threadId }; + await runWorkbench(workbenchOptions, [ + "begin-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + const draft = { + scanId, + complete: true, + findings: [ + { + ruleId: "path-traversal.archive", + title: "Unsafe archive extraction", + summary: + "An untrusted entry reaches a filesystem write.", + severity: { level: "high" }, + confidence: { + level: "high", + rationale: "Source evidence.", + }, + taxonomy: { + category: "path-traversal", + cwe: ["CWE-22"], + }, + locations: [{ path: "extract.py", startLine: 1 }], + remediation: + "Validate the resolved destination before writing.", + provenance: { + source: "local_plugin", + candidateId: "archive-entry", + }, + }, + ], + coverage: { + completeness: "partial", + surfaces: [], + explicitExclusions: [], + deferred: [ + { + id: "dependency", + reason: "A dependency remains unreviewed.", + }, + ], + }, + }; + const seeded = JSON.parse( + execFileSync( + "python3", + [ + fileURLToPath( + new URL( + "./fixtures/selected-deep-scan.py", + import.meta.url, + ), + ), + ], + { + input: JSON.stringify({ + scanId, + scanDir, + database: join(stateDir, "workbench.sqlite3"), + draft, + }), + encoding: "utf8", + env: environment, + }, + ), + ); + // Exercise the dedicated function bridge, without extending CLI arguments. + const selectionOutput = execFileSync( + "python3", + [ + "-c", + "import runpy, sys; script = sys.argv.pop(1); runpy.run_path(script)['main'](select_finalization=True)", + join(PLUGIN_ROOT, "scripts/workbench_db.py"), + "finish-deep-scan", + "--scan-id", + scanId, + "--coordinator-generation", + "2", + "--terminal-reason", + loseCompletionResponse ? "capped" : "saturated", + "--manifest-path", + join(scanDir, "scan-manifest.json"), + ], + { + input: JSON.stringify({ + resultPath: seeded.resultPath, + }), + encoding: "utf8", + env: environment, + }, + ); + originalFinalizationInput = + JSON.parse(selectionOutput).deepScan.finalizationInput; + selectedPath = join( + scanDir, + (originalFinalizationInput as { resultPath: string }) + .resultPath, + ); + selectedBytes = await readFile(selectedPath); + const sessions = join( + codexHome, + "sessions", + "2026", + "01", + "01", + ); + await mkdir(sessions, { recursive: true }); + await writeFile( + join(sessions, `rollout-${threadId}.jsonl`), + JSON.stringify({ + timestamp: new Date().toISOString(), + type: "session_meta", + payload: { id: threadId, cwd: scanDir }, + }) + "\n", + ); + if (ownerSuffix) { + await recordBudgetUsage(); + await resumeSelectedDeepScan({ + scanId, + threadId, + pluginRoot: PLUGIN_ROOT, + runWorkbench: (args) => + runWorkbench(workbenchOptions, args), + signal: cancellation.signal, + }); + await rejoinSelectedScanThroughMcp( + environment, + scanId, + outcome === "completed-owner-explicit", + ); + const saved = await runWorkbench(workbenchOptions, [ + "get-scan", + "--scan-id", + scanId, + ]); + const scan = saved["scan"] as { + progress: { status: string }; + executionAttribution: { completedAt: string | null }; + }; + nativeOwnerStatus = scan.progress.status; + const completedAt = scan.executionAttribution.completedAt; + await recordBudgetUsage( + 2_500, + 60, + new Date( + Math.max( + Date.now(), + completedAt ? Date.parse(completedAt) + 1 : 0, + ), + ).toISOString(), + ); + } + if (outcome === "canceled-before-publication") + cancellation.abort("Synthetic user cancellation"); + if (outcome === "completed" || ownerSuffix) { + yield { + type: "turn.completed", + usage: { + input_tokens: ownerSuffix ? 2_500 : 0, + cached_input_tokens: ownerSuffix ? 200 : 0, + cache_write_input_tokens: 0, + reasoning_output_tokens: 0, + output_tokens: ownerSuffix ? 60 : 0, + }, + }; + } else { + throw parentError; + } + } + return { events: events() }; + }, + }; + return { + startThread: () => thread, + resumeThread: (id: string) => { + expect(id).toBe(threadId); + return thread; + }, + }; + }, + }, + ); + let client = makeClient(); + // Native usage polling is unref'ed; the transport double has no child process. + const keepAlive = setTimeout(() => {}, 30_000); + try { + if (restart) { + await expect( + client.run(repository, { + mode: "deep", + postScanPrompt: resumedStop ? undefined : followUp, + ...(budgeted ? { maxCostUsd: 0.004 } : {}), + }), + ).rejects.toThrow("Synthetic publication write failure"); + const pending = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(pending["deepScan"]).toMatchObject({ + status: "running", + terminalReason: "saturated", + }); + expect(commands).not.toContain("fail-scan"); + await client.close(); + client = makeClient(); + if (initialResumeUsage) await recordBudgetUsage(); + } + if (budgeted || closed) { + const running = client.run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(budgeted ? { maxCostUsd: 0.004 } : {}), + ...(resumedStop ? { resumeScanId: scanId, outputDir: scanDir } : {}), + ...(unpricedUsage + ? { onCost: (cost) => reportedCosts.push(cost.estimatedUsd) } + : {}), + postScanPrompt: followUp, + }); + if (budgeted) { + if ( + budgetCompletionFault === "before-commit" || + budgetCompletionFault === "lost-and-canceled" + ) { + await expect(running).rejects.toBeInstanceOf( + ScanCostLimitExceededError, + ); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + expect(commands).not.toContain("complete-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(saved["scan"]).toMatchObject({ + progress: { + status: + budgetCompletionFault === "before-commit" + ? "failed" + : "complete", + }, + findingCount: 1, + reportAvailable: true, + }); + if (budgetCompletionFault === "lost-and-canceled") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + return; + } + const result = await running; + expect(result.coverage.completeness).toBe("partial"); + expect(JSON.stringify(result.coverage)).toContain("cost limit"); + if (unpricedUsage) { + expect(result.cost).toBeNull(); + expect(reportedCosts).toEqual([]); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect( + (saved["scan"] as { cost?: unknown }).cost ?? null, + ).toBeNull(); + } else expect(result.cost?.estimatedUsd).toBeGreaterThan(0.004); + expect(result.threadId).toBe(threadId); + expect(modelInputs).toHaveLength(1); + expect(commands).toContain("complete-budget-exhausted-scan"); + expect(commands).not.toContain("fail-scan"); + expect( + commands.filter( + (command) => command === "complete-budget-exhausted-scan", + ), + ).toHaveLength(1); + if (budgetCompletionFault === "lost") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + } + const completed = await runWorkbench(workbenchOptions!, [ + "get-deep-scan", + "--scan-id", + scanId, + "--thread-id", + threadId, + ]); + expect(completed["deepScan"]).toMatchObject({ + status: "succeeded", + finalizationInput: { terminalReason: "saturated" }, + }); + return; + } + await expect(running).rejects.toThrow(/closed/); + await closePromise; + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "failed" }, + findingCount: 1, + reportAvailable: true, + }); + expect(deep["deepScan"]).toMatchObject({ + status: "failed", + finalizationInput: { terminalReason: "saturated" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBe("partial"); + expect(commands).toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + await client.close(); + client = makeClient(); + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs).toHaveLength(1); + return; + } + if (outcome.startsWith("canceled-")) { + const error = await client + .run(repository, { + mode: "deep", + signal: cancellation.signal, + ...(resumedStop + ? { resumeScanId: scanId, outputDir: scanDir } + : {}), + postScanPrompt: followUp, + ...(initialResumeUsage ? { maxCostUsd: 0.004 } : {}), + }) + .catch((error: unknown) => error); + if (initialResumeUsage) { + expect(error).toBeInstanceOf(ScanCostLimitExceededError); + expect(error).toBe(originalResumeSignal?.reason); + } else { + expect(error).toBeInstanceOf(ScanInterruptedError); + expect((error as ScanInterruptedError).cause).toBe( + outcome === "canceled-before-publication" + ? parentError + : cancellation.signal.reason, + ); + } + expect(await readFile(selectedPath)).toEqual(selectedBytes!); + if (cancellationReadLost) { + expect(lostCancellationDeepState).toMatchObject({ + status: "running", + finalizationInput: { terminalReason: "saturated" }, + }); + } + expect(originalFinalizationInput).toMatchObject({ + terminalReason: "saturated", + }); + const stopped = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + const deep = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-deep-scan", "--scan-id", scanId, "--thread-id", threadId], + ); + expect(stopped["scan"]).toMatchObject({ + progress: { status: "canceled" }, + findingCount: 1, + reportAvailable: true, + }); + expect( + JSON.parse(await readFile(join(scanDir, "coverage.json"), "utf8")) + .completeness, + ).toBe("partial"); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(deep["deepScan"]).toMatchObject({ + status: "canceled", + }); + expect( + (deep["deepScan"] as Record)["finalizationInput"], + ).toEqual(originalFinalizationInput); + expect( + commands.filter((command) => command === "cancel-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs.length).toBe(1); + if (cancellationFault === "cancel-response") { + expect( + await Promise.all( + ["report.md", "scan-manifest.json"].map((name) => + readFile(join(scanDir, name)), + ), + ), + ).toEqual(completedArtifacts); + } + await expect( + client.run(repository, { + mode: "deep", + resumeScanId: scanId, + outputDir: scanDir, + }), + ).rejects.toThrow(); + expect(modelInputs.length).toBe(1); + return; + } + if (canceledFollowUp) { + await expect( + client.run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: followUp, + }), + ).rejects.toThrow(/interrupted/); + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + findingCount: 1, + reportAvailable: true, + }); + expect(modelInputs.length).toBe(2); + expect(modelInputs[1]).toBe(followUp); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(loseCompletionResponse ? 2 : 1); + expect(commands).not.toContain("cancel-scan"); + expect(commands).not.toContain("fail-scan"); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toBe( + acceptedReport, + ); + return; + } + if (outcome === "completion-before-commit-fails") { + await expect(client.run(repository, { mode: "deep" })).rejects.toThrow( + "Synthetic completion failure before commit", + ); + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(1); + expect(commands).not.toContain("fail-scan"); + expect(modelInputs).toHaveLength(1); + const saved = await runWorkbench(workbenchOptions!, [ + "get-scan", + "--scan-id", + scanId, + ]); + expect(saved["scan"]).toMatchObject({ + progress: { status: "running" }, + }); + return; + } + const result = await client.run(repository, { + mode: "deep", + signal: cancellation.signal, + postScanPrompt: + outcome === "published-before-cancellation" ? undefined : followUp, + ...(restart ? { resumeScanId: scanId, outputDir: scanDir } : {}), + }); + expect(result.threadId).toBe(threadId); + if (outcome === "lost-completion-response") { + expect( + commands.filter((command) => command === "complete-scan"), + ).toHaveLength(2); + } + if (ownerSuffix) { + expect(nativeOwnerStatus).toBe("running"); + expect(result.cost?.inputTokens).toBe(2_500); + expect(result.cost?.outputTokens).toBe(60); + } else { + // The synthetic accepted workers have no native usage receipts. + expect(result.cost).toBeNull(); + } + expect(result.coverage.completeness).toBe("partial"); + expect(result.findings.findings[0]?.remediation).toBe( + "Validate the resolved destination before writing.", + ); + // postScanPrompt retains its existing behavior on each caller invocation. + expect(modelInputs.filter((input) => input !== followUp).length).toBe(1); + expect(modelInputs.filter((input) => input === followUp).length).toBe( + outcome === "published-before-cancellation" ? 0 : restart ? 2 : 1, + ); + const completed = await runWorkbench( + { ...workbenchOptions!, signal: undefined }, + ["get-scan", "--scan-id", scanId], + ); + expect(completed["scan"]).toMatchObject({ + progress: { status: "complete" }, + }); + expect(await readFile(join(scanDir, "report.md"), "utf8")).toContain( + "Validate the resolved destination", + ); + expect(commands).not.toContain("fail-scan"); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }; + test(name, runCase, 30_000); +} + +async function rejoinSelectedScanThroughMcp( + environment: NodeJS.ProcessEnv, + scanId: string, + explicitCompletion = false, +): Promise { + const child = spawn( + process.execPath, + [join(PLUGIN_ROOT, "mcp/server.mjs"), "--stdio"], + { + env: environment, + stdio: ["pipe", "pipe", "pipe"], + }, + ); + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + const exited = new Promise((resolve) => + child.once("exit", () => resolve()), + ); + const result = new Promise((resolve, reject) => { + let buffer = ""; + child.once("error", reject); + child.once("exit", () => + reject(new Error(`MCP closed before selected rejoin: ${stderr}`)), + ); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + buffer += chunk; + for (;;) { + const newline = buffer.indexOf("\n"); + if (newline < 0) break; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + try { + const response = JSON.parse(line) as { + id?: number; + error?: unknown; + result?: { + isError?: boolean; + structuredContent?: { manifestPath?: string }; + }; + }; + if (response.id === 1) { + child.stdin.write( + JSON.stringify({ + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: explicitCompletion + ? "complete_codex_security_scan" + : "start_codex_security_deep_scan", + arguments: { scanId }, + _meta: { + "openai/threadId": threadId, + "codex/sandbox-state-meta": { + permissionProfile: { + type: "managed", + file_system: { + type: "restricted", + entries: [ + { + path: { + type: "special", + value: { kind: "root" }, + }, + access: "read", + }, + ], + }, + network: "restricted", + }, + sandboxCwd: pathToFileURL(PLUGIN_ROOT).href, + }, + }, + }, + }) + "\n", + ); + } + if (response.id === 2) { + expect(response.error).toBeUndefined(); + expect(response.result?.isError).toBeUndefined(); + if (!explicitCompletion) + expect( + response.result?.structuredContent?.manifestPath, + ).toBeDefined(); + resolve(); + } + } catch (error) { + reject(error); + } + } + }); + }); + child.stdin.write( + JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "sdk-owner-completion", version: "0.1.0" }, + }, + }) + "\n", + ); + try { + await result; + } finally { + child.stdin.end(); + child.kill(); + await exited; + } +} diff --git a/sdk/typescript/tests-ts/deep-scan-coverage.test.ts b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts new file mode 100644 index 000000000..4d4996bad --- /dev/null +++ b/sdk/typescript/tests-ts/deep-scan-coverage.test.ts @@ -0,0 +1,114 @@ +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "bun:test"; +import { main } from "../src/cli.js"; +import { loadContract } from "../src/contract.js"; +import { ScanResult } from "../src/result.js"; +import { capture, dependencies } from "./cli-fixtures.js"; + +const fixtureUrl = new URL( + "../../../plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.mjs", + import.meta.url, +); + +test.each([ + ["partial", false, false], + ["unknown", false, false], + ["complete", false, false], + ["partial", true, false], + ["partial", true, true], +] as const)( + "publishes %s source coverage through CLI results (resume: %p, continued: %p)", + async (completeness, resume, continueAfterResume) => { + const root = await mkdtemp(join(tmpdir(), "deep-coverage-publication-")); + try { + await mkdir(join(root, "fixture"), { mode: 0o700 }); + // Keep the real workbench outside other suites' persistent module mocks. + const child = Bun.spawn( + [ + Bun.which("node")!, + fileURLToPath(fixtureUrl), + join(root, "fixture"), + completeness, + String(resume), + String(continueAfterResume), + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [output, errors, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect(exitCode, errors).toBe(0); + const { scanDir, threadId, terminal } = JSON.parse(output); + const contract = await loadContract(scanDir, { + pluginRoot: fileURLToPath( + new URL("../../../plugins/codex-security/", import.meta.url), + ), + }); + const result = new ScanResult({ + ...contract, + scanDir, + threadId, + turnResult: { status: "completed" }, + }); + expect(result.coverage.completeness).toBe(completeness); + const coverage = JSON.parse( + await readFile(join(scanDir, "coverage.json"), "utf8"), + ); + expect(coverage.reviews[0].attempt).toBe(2); + const report = await readFile(join(scanDir, "report.md"), "utf8"); + expect(report).toContain(`| Coverage | ${completeness} |`); + expect(coverage.explicitExclusions).toHaveLength(coverage.reviews.length); + for (const review of coverage.reviews) + expect(report).toContain(review.workerId); + if (completeness === "partial") { + expect( + coverage.deferred.map((item: { reason: string }) => item.reason), + ).toEqual(["Verify entry boundaries.", "Verify symbolic links."]); + expect( + new Set(coverage.deferred.map((item: { id: string }) => item.id)) + .size, + ).toBe(2); + expect( + coverage.reviews.map( + (review: { completeness: string }) => review.completeness, + ), + ).toEqual(["partial", "complete", "unknown"]); + for (const item of coverage.deferred) { + expect(item.provenance.candidateId).toBe("candidate-1"); + expect(report).toContain(item.reason); + expect( + coverage.surfaces.some((surface: { id: string }) => + item.surfaceIds.includes(surface.id), + ), + ).toBe(true); + } + } + for (const surface of coverage.surfaces) { + expect( + await readFile(join(scanDir, surface.receiptRefs[0]), "utf8"), + ).toContain("Synthetic review evidence."); + } + const stdout = capture(); + const stderr = capture(); + expect( + await main( + ["scan", "--mode", "deep", "--json"], + stdout.stream, + stderr.stream, + dependencies({ result, onWorkbench: () => ({ deepScan: terminal }) }), + ), + ).toBe(completeness === "complete" ? 0 : 2); + expect(JSON.parse(stdout.text()).coverage).toEqual(coverage); + if (completeness !== "complete") + expect(stderr.text()).toContain("STOPPED"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index 1df7f704a..8c67de03d 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -34,12 +34,12 @@ const deepScanOwnershipProbe = [ "connection.executescript('''", "CREATE TABLE workspaces (id TEXT PRIMARY KEY, thread_id TEXT, updated_at TEXT);", "CREATE TABLE scans (id TEXT PRIMARY KEY, workspace_id TEXT, mode TEXT, status TEXT, recipe_json TEXT, handoff_status TEXT, handoff_claim_token TEXT, deep_scan_owner_thread_id TEXT, updated_at TEXT);", - "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY);", + "CREATE TABLE deep_scan_runs (scan_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL DEFAULT 1, workflow_version TEXT NOT NULL DEFAULT 'deep-scan-mcp/v1');", "''')", "scan_id = '11111111-1111-4111-8111-111111111111'", "connection.execute(\"INSERT INTO workspaces VALUES ('workspace', NULL, 'before')\")", "connection.execute(\"INSERT INTO scans VALUES (?, 'workspace', 'deep', 'running', '{}', 'delivered', ?, NULL, 'before')\", (scan_id, case['storedToken']))", - "connection.execute('INSERT INTO deep_scan_runs VALUES (?)', (scan_id,))", + "connection.execute('INSERT INTO deep_scan_runs (scan_id) VALUES (?)', (scan_id,))", "connection.commit()", "if case.get('mutation') == 'rotate':", " connection.executescript(\"CREATE TRIGGER rotate_claim BEFORE UPDATE OF thread_id ON workspaces BEGIN UPDATE scans SET handoff_claim_token = '33333333-3333-4333-8333-333333333333' WHERE workspace_id = NEW.id; END\")", @@ -908,9 +908,7 @@ describe("deep scan workbench ownership", () => { ]), ); const report = await readFile(join(scanDir, "report.md"), "utf8"); - expect(report).toContain( - "No findings were validated before the scan reached its cost limit.", - ); + expect(report).toContain("partial report"); if (existingDeferred) { expect(report).toContain( "Existing candidate validation dependency was unavailable.", @@ -965,7 +963,7 @@ describe("deep scan workbench ownership", () => { false, ], ] as const)( - "only describes an exhausted scan cost limit for %s", + "uses the partial-report summary for an exhausted scan cost limit: %s", (_description, reason, exhausted) => { const python = Bun.which("python3") ?? Bun.which("python"); expect(python).not.toBeNull(); @@ -987,11 +985,7 @@ describe("deep scan workbench ownership", () => { ); expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); const report = new TextDecoder().decode(result.stdout); - expect( - report.includes( - "No findings were validated before the scan reached its cost limit.", - ), - ).toBe(exhausted); + expect(report.includes("partial report")).toBe(exhausted); expect( report.includes( "No reportable findings survived the canonical discovery, validation, and reportability gates.", diff --git a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts index c23adfb4d..a2a74d36d 100644 --- a/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-worker-shutdown.test.ts @@ -31,6 +31,18 @@ async function bundledWorkerExecutor( if (source === undefined) { throw new Error("Bundled Deep Scan worker executor was not found."); } + const sessionSource = + /\n\/\/ [^\n]*\/codex-session\.ts\n([\s\S]*?)(?=\n\/\/)/u.exec( + runtime, + )?.[1]; + expect(sessionSource).toBeDefined(); + const recordFunction = /\b(isRecord\d*)\(/u.exec(source)?.[1]; + expect(recordFunction).toBeDefined(); + const recordSource = new RegExp( + `function ${recordFunction}\\([^\\n]*\\) \\{[\\s\\S]*?\\n\\}`, + "u", + ).exec(runtime)?.[0]; + expect(recordSource).toBeDefined(); const fileSystemImport = /\b(import_node_fs\d*)\.promises\.readFile\(/u.exec( source, )?.[1]; @@ -54,7 +66,8 @@ async function bundledWorkerExecutor( "workerPermissionProfile", "workerPermissionProfileConfigOverrides", "snapshotWorkerEnvironment", - "workerReasoningSummary", + "workerModelConfig", + "workerModelSelection", "environmentVariable", "preflightDeepScanWorkerPermissionProfile", "DEEP_SCAN_WORKER_PERMISSION_PROFILE_ID", @@ -64,14 +77,15 @@ async function bundledWorkerExecutor( "workerSubagentConfig", "appendSafeItemDiagnostic", "classifyCodexWorkerError", - `${source}\nreturn CodexSdkWorkerExecutor;`, + `${sessionSource}\n${recordSource}\n${source}\nreturn CodexSdkWorkerExecutor;`, )( FakeCodex, { promises: { readFile: async () => "fixture worker prompt" } }, () => ({}), () => [], async () => ({}), - async () => undefined, + async () => ({}), + () => ({}), () => undefined, preflight, "codex_security_deep_scan_worker", diff --git a/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py new file mode 100644 index 000000000..5b741021c --- /dev/null +++ b/sdk/typescript/tests-ts/fixtures/selected-deep-scan.py @@ -0,0 +1,67 @@ +"""Synthetic accepted workers for the installed finalization/SDK contract test.""" +import hashlib +import json +import sqlite3 +import sys +import uuid +from pathlib import Path + +payload = json.load(sys.stdin) +scan_id = payload["scanId"] +scan_dir = Path(payload["scanDir"]) +with sqlite3.connect(payload["database"]) as connection: + connection.execute("PRAGMA foreign_keys = ON") + timestamp = connection.execute( + "SELECT created_at FROM deep_scan_runs WHERE scan_id = ?", (scan_id,) + ).fetchone()[0] + connection.execute( + "UPDATE deep_scan_runs SET workflow_version = 'deep-security-scan/v2', " + "coordinator_generation = 2, phase = 'reducing', " + "discovery_runs_dispatched = 2, completion_sequence = 2, " + "consecutive_no_new = 2, stop_after_no_new = 2, max_discovery_runs = 2 " + "WHERE scan_id = ?", (scan_id,), + ) + discoveries = [] + for kind, label in [("discovery", "review-1"), ("discovery", "review-2"), ("dedup", "merge-1")]: + draft = dict(payload["draft"]) + if kind == "dedup": + draft["sourceCoverage"] = draft.pop("coverage") + encoded = json.dumps(draft).encode() + digest = hashlib.sha256(encoded).hexdigest() + worker_id = str(uuid.uuid4()) + output = scan_dir / "artifacts" / "deep_discovery" / label / "output" + output.mkdir(parents=True) + prompt = output.parent / "prompt.md" + prompt.write_text("Synthetic accepted audit\n") + accepted = output / "checkpoints" / f"{digest}.json" + accepted.parent.mkdir() + accepted.write_bytes(encoded) + result = output / "result.json" + result.write_bytes(encoded) + sequence = len(discoveries) + 1 if kind == "discovery" else None + connection.execute( + "INSERT INTO deep_scan_workers " + "(id, scan_id, kind, status, merge_state, prompt_path, artifact_dir, " + "result_manifest_path, attempt, completion_sequence, created_at, updated_at, completed_at) " + "VALUES (?, ?, ?, 'succeeded', ?, ?, ?, ?, 1, ?, ?, ?, ?)", + (worker_id, scan_id, kind, "merged" if kind == "discovery" else "none", + str(prompt), str(output), str(result), sequence, timestamp, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO deep_scan_attempts " + "(scan_id, worker_id, attempt, status, started_at, completed_at, accepted_result_path, accepted_result_sha256) " + "VALUES (?, ?, 1, 'succeeded', ?, ?, ?, ?)", + (scan_id, worker_id, timestamp, timestamp, str(accepted), digest), + ) + if kind == "discovery": + discoveries.append(worker_id) + else: + for order, discovery in enumerate(discoveries): + connection.execute( + "INSERT INTO deep_scan_dedup_inputs " + "(scan_id, dedup_worker_id, discovery_worker_id, input_order) VALUES (?, ?, ?, ?)", + (scan_id, worker_id, discovery, order), + ) + # The committed immutable reference survives loss of the replaceable output. + result.unlink() +print(json.dumps({"resultPath": str(result), "acceptedPath": str(accepted)})) diff --git a/sdk/typescript/tests-ts/reasoning-summary.test.ts b/sdk/typescript/tests-ts/reasoning-summary.test.ts new file mode 100644 index 000000000..854ee85ea --- /dev/null +++ b/sdk/typescript/tests-ts/reasoning-summary.test.ts @@ -0,0 +1,82 @@ +import { execFileSync } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, test } from "bun:test"; +import { captureOriginalReasoningSummary } from "../src/reasoning-summary.js"; +import type { JsonObject } from "../src/config.js"; + +test.each([ + { model_reasoning_summary: "none" }, + { model_reasoning_summary: "concise" }, + { model_reasoning_summary: null }, + { model_reasoning_summary: "" }, + { + profile: "selected", + profiles: { selected: { model_reasoning_summary: "auto" } }, + }, +] as JsonObject[])( + "preserves explicit summary without a native lookup: %j", + async (config) => { + expect( + await captureOriginalReasoningSummary({ + config, + command: { command: "unused-native-executable" }, + cwd: tmpdir(), + environment: {}, + signal: new AbortController().signal, + }), + ).toBeUndefined(); + }, +); + +test.each(["unknown-model", "unsupported-command", "missing-metadata"])( + "keeps an unavailable original model default unknown: %s", + async (scenario) => { + const root = await mkdtemp(join(tmpdir(), "summary-selection-")); + try { + const cwd = join(root, "original output"); + const script = join(root, "native.mjs"); + const receipt = join(root, "receipt.json"); + await mkdir(cwd); + await writeFile( + script, + [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(receipt)}, JSON.stringify({cwd:process.cwd(),args:process.argv,home:process.env.CODEX_HOME}));`, + `console.log(JSON.stringify({models:[{slug:${JSON.stringify(scenario === "unknown-model" ? "another-model" : "selected-model")}${scenario === "missing-metadata" ? "" : ',default_reasoning_summary:"none"'}}]}));`, + `process.exit(${scenario === "unsupported-command" ? 1 : 0});`, + ].join("\n"), + ); + const command = execFileSync("node", ["-p", "process.execPath"], { + encoding: "utf8", + }).trim(); + const value = await captureOriginalReasoningSummary({ + config: { model: "selected-model", model_reasoning_effort: "high" }, + command: { command }, + cwd, + environment: { + CODEX_HOME: root, + NODE_OPTIONS: `--import=${pathToFileURL(script).href}`, + }, + signal: new AbortController().signal, + }); + expect(value).toBeUndefined(); + const recorded = JSON.parse(await readFile(receipt, "utf8")); + expect(await realpath(recorded.cwd)).toBe(await realpath(cwd)); + expect(recorded.home).toBe(root); + expect(recorded.args).toContain('model="selected-model"'); + expect(recorded.args).toContain('model_reasoning_effort="high"'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, +); diff --git a/sdk/typescript/tests-ts/scan-logs.test.ts b/sdk/typescript/tests-ts/scan-logs.test.ts index 059e0e42f..ed3756ff3 100644 --- a/sdk/typescript/tests-ts/scan-logs.test.ts +++ b/sdk/typescript/tests-ts/scan-logs.test.ts @@ -82,6 +82,274 @@ function commandEvent(command: string, id: string, timestamp?: string) { } describe("saved scan logs", () => { + test.each( + ["sessions", "archived_sessions"].flatMap((directory) => + ["prefix only", "unrelated owner turn"].map((tail) => [directory, tail]), + ), + )( + "reads the complete same-thread recorded copy from %s after %s", + async (directory, tail) => { + const current = await temporaryHome(); + const original = await temporaryHome(); + const scanDir = await temporaryHome(); + const settings = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settings, { recursive: true }); + await writeFile( + join(settings, "execution-settings.json"), + JSON.stringify({ version: 1, settings: { codexHome: original } }), + ); + const timestamp = "2026-08-11T12:01:00.000Z"; + const prefix = [ + { type: "turn_context", timestamp, payload: { turn_id: "scan-turn" } }, + commandEvent("prefix", "prefix-call", timestamp), + ]; + const suffix = commandEvent("saved suffix", "suffix-call", timestamp); + await writeSession(current, "owner", [ + ...prefix, + ...(tail === "unrelated owner turn" + ? [ + { + type: "turn_context", + timestamp, + payload: { turn_id: "other-turn" }, + }, + commandEvent( + "unrelated first copy", + "other-first-call", + timestamp, + ), + ] + : []), + ]); + const currentPath = join( + current, + "sessions", + "2026", + "08", + "11", + "rollout-owner.jsonl", + ); + await writeFile( + currentPath, + (await readFile(currentPath, "utf8")) + '\n\n42\n{"type":', + ); + await writeSession(original, "owner", [ + ...prefix, + suffix, + suffix, + { type: "turn_context", timestamp, payload: { turn_id: "other-turn" } }, + commandEvent("unrelated turn", "other-call", timestamp), + ]); + if (directory === "archived_sessions") { + await rename(join(original, "sessions"), join(original, directory)); + } + const result = await readSavedScanLogs( + { + scanId: "scan-1", + mode: "deep", + scanDir, + continuationThreadId: "owner", + executionAttribution: { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "owner", + turnId: "scan-turn", + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }, + current, + ); + expect(result.sessions.map(({ threadId }) => threadId)).toEqual([ + "owner", + ]); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + ...prefix, + suffix, + suffix, + ]); + expect(result.sessions[0]?.path).toStartWith(join(original, directory!)); + }, + ); + + test.each(["equal", "shorter", "divergent"])( + "keeps the first rollout when attributed occurrences are %s", + async (copy) => { + const first = await temporaryHome(); + const second = await temporaryHome(); + const timestamp = "2026-08-11T12:01:00.000Z"; + const turn = { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }; + const prefix = commandEvent("first", "first-call", timestamp); + const suffix = commandEvent("complete", "complete-call", timestamp); + const unrelated = { + type: "turn_context", + timestamp, + payload: { turn_id: "other-turn" }, + }; + await writeSession(first, "owner", [turn, prefix, suffix, unrelated]); + await writeSession(second, "owner", [ + turn, + ...(copy === "equal" + ? [prefix, suffix] + : copy === "shorter" + ? [prefix] + : [ + prefix, + commandEvent("different", "different-call", timestamp), + suffix, + ]), + unrelated, + commandEvent("more unrelated work", "other-call", timestamp), + ]); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "owner", + codexHome: [first, second], + executionAttribution: { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "owner", + turnId: "scan-turn", + startedAt: timestamp, + }, + startedAt: timestamp, + completedAt: timestamp, + }, + }); + expect(result.sessions[0]?.path).toStartWith(first); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + turn, + prefix, + suffix, + ]); + }, + ); + + test.each(["equal", "shorter", "divergent"])( + "keeps the first rollout when a later copy is %s", + async (copy) => { + const first = await temporaryHome(); + const second = await temporaryHome(); + const prefix = commandEvent("first", "first-call"); + const suffix = commandEvent("complete", "complete-call"); + await writeSession(first, "owner", [prefix, suffix]); + await writeSession( + second, + "owner", + copy === "equal" + ? [prefix, suffix] + : copy === "shorter" + ? [prefix] + : [prefix, commandEvent("different", "different-call"), suffix], + ); + const result = await readScanLogs({ + scanId: "scan-1", + threadId: "owner", + codexHome: [first, second], + }); + expect(result.sessions[0]?.path).toStartWith(first); + expect(result.events.map(({ event }) => event)).toEqual([ + { type: "session_meta", payload: { id: "owner" } }, + prefix, + suffix, + ]); + }, + ); + + test.each(["current", "original"])( + "reads recovered Deep workers from the recorded home with the owner in the %s home", + async (ownerHome) => { + const current = await temporaryHome(); + const original = await temporaryHome(); + const scanDir = await temporaryHome(); + const settingsDirectory = join(scanDir, "artifacts", "deep_discovery"); + await mkdir(settingsDirectory, { recursive: true }); + await writeFile( + join(settingsDirectory, "execution-settings.json"), + JSON.stringify({ + version: 1, + settings: { codexHome: original, codexPath: join(original, "codex") }, + }), + ); + const startedAt = "2026-08-11T12:00:00.000Z"; + const timestamp = "2026-08-11T12:01:00.000Z"; + await writeSession( + ownerHome === "current" ? current : original, + "owner", + [ + { + type: "turn_context", + timestamp, + payload: { turn_id: "scan-turn" }, + }, + commandEvent("scan owner", "owner-call", timestamp), + { + type: "turn_context", + timestamp, + payload: { turn_id: "later-turn" }, + }, + commandEvent("unrelated owner turn", "other-turn", timestamp), + ], + ); + for (const id of ["discovery", "resumed-discovery", "reducer"]) { + await writeSession(original, id, [commandEvent(id, id, timestamp)]); + } + await writeSession( + original, + "worker-child", + [commandEvent("worker child", "child-call", timestamp)], + "discovery", + ); + // Keep the existing first-home preference for duplicate active rollouts. + await writeSession(current, "reducer", [ + commandEvent("current reducer", "current-reducer-call", timestamp), + ]); + for (const home of [current, original]) { + await writeSession(home, "unrelated", [ + commandEvent("unrelated scan", "unrelated-call", timestamp), + ]); + } + const result = await readSavedScanLogs( + { + scanId: "scan-1", + mode: "deep", + scanDir, + continuationThreadId: "owner", + executionAttribution: { + formatVersion: 1, + executionThreadIds: ["discovery", "resumed-discovery", "reducer"], + owner: { threadId: "owner", turnId: "scan-turn", startedAt }, + startedAt, + completedAt: "2026-08-11T12:02:00.000Z", + }, + }, + current, + ); + expect(result.sessions.map(({ threadId }) => threadId).sort()).toEqual([ + "discovery", + "owner", + "reducer", + "resumed-discovery", + "worker-child", + ]); + expect( + result.events.filter(({ threadId }) => threadId === "reducer"), + ).toHaveLength(2); + expect(JSON.stringify(result)).toContain("current reducer"); + expect(JSON.stringify(result)).not.toContain("unrelated"); + }, + ); + test("collects known desktop and CLI threads across active and archived homes without duplicates", async () => { const desktop = await temporaryHome(); const cli = await temporaryHome(); @@ -304,6 +572,7 @@ describe("saved scan logs", () => { test("excludes inherited parent history from worker logs", async () => { const home = await temporaryHome(); + const current = await temporaryHome(); await writeSession(home, "parent", []); const startedAt = "2026-08-11T12:02:00.900Z"; await writeSession( @@ -347,10 +616,31 @@ describe("saved scan logs", () => { startedAt, ); + const path = join( + home, + "sessions", + "2026", + "08", + "11", + "rollout-worker.jsonl", + ); + const currentPath = join(current, "sessions", "rollout-worker.jsonl"); + await mkdir(join(current, "sessions"), { recursive: true }); + await writeFile( + currentPath, + (await readFile(path, "utf8")) + .split("\n") + .slice(0, 4) + .join("\n") + .replace( + "PRIVATE PRE-SCAN CONVERSATION", + "OTHER PRE-SCAN CONVERSATION", + ), + ); const result = await readScanLogs({ scanId: "scan-1", threadId: "parent", - codexHome: home, + codexHome: [current, home], }); expect(JSON.stringify(result)).toContain("Reviewing authorization"); expect(JSON.stringify(result)).not.toContain("PRIVATE PRE-SCAN"); @@ -525,12 +815,17 @@ describe("saved scan logs", () => { test("does not parse event bodies from unrelated saved sessions", async () => { const home = await temporaryHome(); + const other = await temporaryHome(); await writeSession(home, "parent", [ commandEvent("included", "parent-call"), ]); await writeSession(home, "unrelated", [ commandEvent("UNRELATED_PRIVATE_EVENT_BODY", "unrelated-call"), ]); + await writeSession(other, "unrelated", [ + commandEvent("UNRELATED_PRIVATE_EVENT_BODY", "unrelated-call"), + commandEvent("UNRELATED_PRIVATE_EVENT_BODY", "later-call"), + ]); const originalParse = JSON.parse; let unrelatedBodies = 0; const parseSpy = spyOn(JSON, "parse").mockImplementation( @@ -544,7 +839,7 @@ describe("saved scan logs", () => { const result = await readScanLogs({ scanId: "scan-1", threadId: "parent", - codexHome: home, + codexHome: [home, other], }); expect(result.sessions.map(({ threadId }) => threadId)).toEqual([ "parent", diff --git a/sdk/typescript/tests-ts/scan-resume.test.ts b/sdk/typescript/tests-ts/scan-resume.test.ts index a70986bb2..d2cf81023 100644 --- a/sdk/typescript/tests-ts/scan-resume.test.ts +++ b/sdk/typescript/tests-ts/scan-resume.test.ts @@ -158,10 +158,16 @@ async function interruptedScan( const sessionPath = join(codexHome, "sessions", `rollout-${threadId}.jsonl`); await writeFile( sessionPath, - JSON.stringify({ - type: "session_meta", - payload: { id: threadId, cwd: scanDir }, - }) + "\n", + [ + { type: "session_meta", payload: { id: threadId, cwd: scanDir } }, + { + type: "turn_context", + timestamp: new Date().toISOString(), + payload: { turn_id: "synthetic-scan-turn", model: "gpt-5.6-sol" }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n", ); if (mode === "deep") { await command([ @@ -488,6 +494,7 @@ test.each([ f.sessionPath, JSON.stringify({ type: "event_msg", + timestamp: new Date().toISOString(), payload: { type: "token_count", info: { diff --git a/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts new file mode 100644 index 000000000..6265ebf0b --- /dev/null +++ b/sdk/typescript/tests-ts/scan-usage-reconciliation.test.ts @@ -0,0 +1,601 @@ +import { describe, expect, test } from "bun:test"; +import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { estimateScanCost } from "../src/cost-model.js"; +import { ScanCostTracker } from "../src/cost.js"; +import { readScanLogs } from "../src/scan-logs.js"; +import type { ScanExecutionAttribution } from "../src/scan-sessions.js"; + +describe("scan usage reconciliation", () => { + test("SDK usage and logs share attempt membership and the original owner turn", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-attribution-")); + const observed: unknown[] = []; + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + onSessionEvent: (event) => observed.push(event), + }); + const at = (second: number) => + `2026-09-01T00:00:${String(second).padStart(2, "0")}Z`; + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: ["old-worker", "replacement-worker"], + owner: { threadId: "parent", turnId: "scan-turn", startedAt: at(1) }, + startedAt: at(1), + completedAt: at(10), + }; + const token = (second: number, count: number) => ({ + timestamp: at(second), + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: count, output_tokens: 0 } }, + }, + }); + const context = (second: number, turn: string) => ({ + timestamp: at(second), + type: "turn_context", + payload: { turn_id: turn, model: "gpt-5.6-sol" }, + }); + try { + await mkdir(join(home, "sessions")); + for (const [id, parent, events] of [ + [ + "parent", + null, + [ + context(0, "prior-turn"), + token(0, 100), + context(1, "scan-turn"), + token(2, 110), + context(3, "unrelated-turn"), + token(4, 1010), + ], + ], + [ + "old-worker", + null, + [context(1, "worker-turn"), token(2, 20), token(11, 120)], + ], + ["replacement-worker", null, [context(3, "worker-turn"), token(4, 30)]], + ["worker-child", "old-worker", [context(3, "child-turn"), token(4, 5)]], + ["unrelated-child", "parent", [context(3, "side-turn"), token(4, 900)]], + ] as const) { + const records = [ + { + type: "session_meta", + payload: { id, ...(parent ? { parent_thread_id: parent } : {}) }, + }, + ...events, + ]; + await writeFile( + join(home, "sessions", `${id}.jsonl`), + records.map((value) => JSON.stringify(value)).join("\n") + "\n", + ); + } + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(65); + expect(JSON.stringify(observed)).not.toContain("unrelated-turn"); + expect(JSON.stringify(observed)).not.toContain(at(11)); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + expect(logs.sessions.map((session) => session.threadId).sort()).toEqual([ + "old-worker", + "parent", + "replacement-worker", + "worker-child", + ]); + expect( + logs.events.some(({ event }) => + JSON.stringify(event).includes("unrelated-turn"), + ), + ).toBe(false); + expect( + logs.events.some(({ event }) => JSON.stringify(event).includes(at(11))), + ).toBe(false); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("waits for attribution and retains uncertainty until missing attempt usage arrives", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + let attribution: ScanExecutionAttribution | null = null; + tracker.setAttributionReader(async () => attribution); + const at = "2026-09-01T00:00:02Z"; + const records = (id: string, count: number) => + [ + { type: "session_meta", payload: { id } }, + { + timestamp: at, + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "own" }, + }, + { + timestamp: at, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: count, output_tokens: 0 }, + }, + }, + }, + ] + .map((event) => JSON.stringify(event)) + .join("\n") + "\n"; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + records("worker", 20), + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 20, output_tokens: 0 }); + expect((await tracker.refresh()).cost).toBeNull(); + attribution = { + formatVersion: 1, + executionThreadIds: ["worker", "failed-attempt"], + owner: { threadId: "worker", turnId: "own", startedAt: at }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 20, + coverage: "partial", + }); + await writeFile( + join(home, "sessions", "failed.jsonl"), + records("failed-attempt", 5), + ); + const final = await tracker.stop(); + expect(final.cost?.inputTokens).toBe(25); + expect(final.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("preserves receipt accounting for a resumed legacy Deep scan", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-legacy-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + tracker.setAttributionReader(async () => ({ + formatVersion: 1, + legacy: true, + executionThreadIds: ["legacy-parent"], + owner: { + threadId: "legacy-parent", + turnId: null, + startedAt: "2026-09-01T00:00:00Z", + }, + startedAt: "2026-09-01T00:00:00Z", + completedAt: null, + })); + try { + tracker.start("legacy-parent"); + const snapshot = await tracker.stop({ + input_tokens: 10000, + output_tokens: 100, + }); + expect(snapshot.cost?.inputTokens).toBe(10000); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("prices each observed model instead of repricing the sum with the parent", () => { + const usage = { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: "gpt-6-astra", input_tokens: 100, output_tokens: 10 }, + ], + }; + const expected = + estimateScanCost("gpt-5.6-sol", usage.modelUsage[0])!.estimatedUsd + + estimateScanCost("gpt-6-astra", usage.modelUsage[1])!.estimatedUsd; + expect(estimateScanCost("gpt-5.6-sol", usage)?.estimatedUsd).toBe(expected); + }); + + test("keeps incomplete model attribution unpriced", () => { + expect( + estimateScanCost("gpt-5.6-sol", { + input_tokens: 200, + output_tokens: 20, + modelUsage: [ + { model: "gpt-5.6-sol", input_tokens: 100, output_tokens: 10 }, + { model: null, input_tokens: 100, output_tokens: 10 }, + ], + }), + ).toBeNull(); + }); + + test("reconciles stale and missing cumulative receipts without reducing usage", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 160, output_tokens: 0 }); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + tracker.recordUsage(null); + expect((await tracker.stop()).cost?.inputTokens).toBe(160); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + + test("tracks per-model deltas within one resumed session", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-models-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + await mkdir(join(home, "sessions")); + const records = [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "turn_context", + payload: { model: "gpt-5.6-sol", turn_id: "turn-1" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 100, output_tokens: 10 }, + }, + }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { input_tokens: 60, output_tokens: 6 } }, + }, + }, + { + type: "turn_context", + payload: { model: "gpt-6-astra", turn_id: "turn-2" }, + }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 200, output_tokens: 20 }, + }, + }, + }, + ]; + await writeFile( + join(home, "sessions", "worker.jsonl"), + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage({ input_tokens: 200, output_tokens: 20 }); + const snapshot = await tracker.stop(); + expect(snapshot.cost?.inputTokens).toBe(200); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.0021, 12); + expect(snapshot.cost?.modelCosts?.map((cost) => cost.model)).toEqual([ + "gpt-5.6-sol", + "gpt-6-astra", + ]); + tracker.recordUsage({ input_tokens: 250, output_tokens: 25 }); + expect((await tracker.refresh()).cost).toBeNull(); + await appendFile( + join(home, "sessions", "worker.jsonl"), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { input_tokens: 250, output_tokens: 25 }, + }, + }, + }) + "\n", + ); + expect((await tracker.refresh()).cost?.estimatedUsd).toBeCloseTo( + 0.00285, + 12, + ); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe("charged response receipts", () => { + for (const exactOnly of [false, true]) { + test(`counts compaction and deduplicates responses across counter resets (exact only: ${exactOnly})`, async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-receipts-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (input: number, cached: number, output: number) => ({ + input_tokens: input, + cached_input_tokens: cached, + cache_write_input_tokens: 0, + output_tokens: output, + reasoning_output_tokens: 0, + total_tokens: input + output, + }); + const record = ( + id: string, + count: unknown, + cumulative: unknown, + model = "gpt-5.6-sol", + ) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + turn_id: "turn", + response_id: id, + model, + usage: count, + thread_token_usage: cumulative, + }, + }); + const first = record("normal-1", usage(100, 80, 10), usage(100, 80, 10)); + const compact = record( + "compaction", + usage(50, 40, 5), + usage(150, 120, 15), + "gpt-6-astra", + ); + const second = record("normal-2", usage(120, 90, 12), usage(120, 90, 12)); + const counter = (count: unknown) => ({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: count }, + }, + }); + const events = [ + { type: "session_meta", payload: { id: "worker" } }, + first, + ...(!exactOnly ? [counter(usage(100, 80, 10))] : []), + compact, + { + type: "compacted", + payload: { message: "Synthetic context summary" }, + }, + compact, + second, + ...(!exactOnly ? [counter(usage(220, 170, 22))] : []), + first, + ]; + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + events.map((e) => JSON.stringify(e)).join("\n") + "\n", + ); + tracker.start("worker"); + tracker.recordUsage(usage(220, 170, 22)); + const result = await tracker.stop(); + expect(result.usage).toMatchObject(usage(270, 210, 27)); + expect( + result.cost?.modelCosts?.map((part) => [ + part.model, + part.inputTokens + part.outputTokens, + ]), + ).toEqual([ + ["gpt-5.6-sol", 242], + ["gpt-6-astra", 55], + ]); + expect((await tracker.refresh()).usage).toEqual(result.usage); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); + } + + test("uses receipt turn identity for shared-parent usage and logs", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-response-owner-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const attribution: ScanExecutionAttribution = { + formatVersion: 1, + executionThreadIds: [], + owner: { + threadId: "parent", + turnId: "scan-turn", + startedAt: "2026-09-01T00:00:01Z", + }, + startedAt: "2026-09-01T00:00:01Z", + completedAt: "2026-09-01T00:00:10Z", + }; + const receipt = ( + id: string, + turn: string, + second: string, + input: number, + ) => ({ + type: "token_usage_record", + timestamp: `2026-09-01T00:00:${second}Z`, + payload: { + response_id: id, + thread_id: "parent", + turn_id: turn, + model: "gpt-5.6-sol", + usage: { input_tokens: input, output_tokens: 0 }, + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "parent.jsonl"), + [ + { type: "session_meta", payload: { id: "parent" } }, + receipt("prior", "prior-turn", "00", 900), + receipt("owned", "scan-turn", "02", 20), + receipt("side", "other-turn", "03", 800), + receipt("post", "scan-turn", "11", 700), + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.setAttributionReader(async () => attribution); + tracker.start("parent"); + expect((await tracker.stop()).cost?.inputTokens).toBe(20); + const logs = await readScanLogs({ + scanId: "scan", + threadId: "parent", + codexHome: home, + executionAttribution: attribution, + }); + const ids = logs.events + .map( + ({ event }) => + (event as { payload?: Record })["payload"]?.[ + "response_id" + ], + ) + .filter(Boolean); + expect(ids).toEqual(["owned"]); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } + }); +}); + +test("delayed response receipts resolve cumulative gaps without treating smaller counters as stale", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-delayed-response-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const record = (id: string, tokens: number, cumulative: number) => + JSON.stringify({ + type: "token_usage_record", + payload: { + response_id: id, + thread_id: "worker", + model: "gpt-5.6-sol", + usage: { input_tokens: tokens, output_tokens: 0 }, + thread_token_usage: { input_tokens: cumulative, output_tokens: 0 }, + }, + }) + "\n"; + try { + await mkdir(join(home, "sessions")); + const file = join(home, "sessions", "worker.jsonl"); + await writeFile( + file, + JSON.stringify({ type: "session_meta", payload: { id: "worker" } }) + + "\n" + + record("first", 100, 100) + + record("third", 50, 180), + ); + tracker.start("worker"); + expect((await tracker.refresh()).cost).toMatchObject({ + inputTokens: 150, + coverage: "partial", + }); + await appendFile(file, record("second", 30, 130)); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(180); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("a reader installed before the native attribution writer preserves legacy receipts", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-reader-first-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + try { + tracker.setAttributionReader(async () => undefined); + tracker.start("parent"); + tracker.recordUsage({ input_tokens: 100, output_tokens: 0 }); + expect((await tracker.stop()).cost?.inputTokens).toBe(100); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); + +test("late exact receipts replace an overlapping legacy counter without adding it twice", async () => { + const home = await mkdtemp(join(tmpdir(), "usage-overlap-")); + const tracker = new ScanCostTracker({ + codexHome: home, + model: "gpt-5.6-sol", + }); + const usage = (count: number) => ({ input_tokens: count, output_tokens: 0 }); + const receipt = (id: string, count: number, cumulative: number) => ({ + type: "token_usage_record", + payload: { + thread_id: "worker", + response_id: id, + model: "gpt-5.6-sol", + usage: usage(count), + thread_token_usage: usage(cumulative), + }, + }); + try { + await mkdir(join(home, "sessions")); + await writeFile( + join(home, "sessions", "worker.jsonl"), + [ + { type: "session_meta", payload: { id: "worker" } }, + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(100) }, + }, + }, + receipt("new", 10, 110), + receipt("old", 100, 100), + { + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage(10) }, + }, + }, + ] + .map((e) => JSON.stringify(e)) + .join("\n") + "\n", + ); + tracker.start("worker"); + const result = await tracker.stop(); + expect(result.cost?.inputTokens).toBe(110); + expect(result.cost?.coverage).toBeUndefined(); + } finally { + await tracker.stop(); + await rm(home, { recursive: true, force: true }); + } +}); diff --git a/sdk/typescript/tests-ts/support/cli-logs-large.mts b/sdk/typescript/tests-ts/support/cli-logs-large.mts new file mode 100644 index 000000000..2ac765fbd --- /dev/null +++ b/sdk/typescript/tests-ts/support/cli-logs-large.mts @@ -0,0 +1,92 @@ +import { constants } from "node:buffer"; +import { createHash } from "node:crypto"; +import { mkdir, open, realpath } from "node:fs/promises"; +import { join } from "node:path"; +import { Writable } from "node:stream"; +import { main } from "../../src/cli.js"; +import { dependencies } from "../cli-fixtures.js"; + +const state = await realpath(process.argv[2]!); +const sessions = join(state, "codex-home", "sessions"); +await mkdir(sessions, { recursive: true }); +const path = join(sessions, "rollout.jsonl"); +const metadata = { type: "session_meta", payload: { id: "thread-1" } }; +let payload: unknown = Array(4096).fill(0); +// Pretty indentation expands a small input into a large document economically. +for (let i = 0; i < 32; i++) payload = { nested: payload }; +const event = (sequence: number) => ({ type: "event_msg", sequence, payload }); +const header = { + scanId: "scan-1", + threadId: "thread-1", + sessions: [{ threadId: "thread-1", parentThreadId: null, path }], + events: [{ threadId: "thread-1", event: metadata }], +}; +const first = JSON.stringify(header, null, 2); +const prefix = first.slice(0, first.lastIndexOf("\n ]")); +const suffix = "\n ]\n}\n"; +const record = (sequence: number) => + JSON.stringify( + { events: [{ threadId: "thread-1", event: event(sequence) }] }, + null, + 2, + ).slice('{\n "events": [\n'.length, -"\n ]\n}".length); +const count = Math.ceil(constants.MAX_STRING_LENGTH / record(0).length) + 1; +const expected = createHash("sha256"); +let expectedBytes = 0; +function expectChunk(text: string) { + expected.update(text); + expectedBytes += Buffer.byteLength(text); +} +expectChunk(prefix); +const file = await open(path, "wx"); +try { + await file.write(`${JSON.stringify(metadata)}\n`); + for (let i = 0; i < count; i++) { + await file.write(`${JSON.stringify(event(i))}\n`); + expectChunk(`,\n${record(i)}`); + } +} finally { + await file.close(); +} +expectChunk(suffix); +let bytes = 0; +const actual = createHash("sha256"); +const output = new Writable({ + write(chunk, _encoding, callback) { + bytes += chunk.length; + actual.update(chunk); + callback(); + }, +}); +const deps = dependencies({ + environment: { CODEX_SECURITY_STATE_DIR: state }, + onWorkbench: () => ({ + scan: { scanId: "scan-1", continuationThreadId: "thread-1" }, + }), +}); +deps.createSecurity = () => { + throw new Error("Reading logs must not start Codex"); +}; +try { + const exitCode = await main( + ["scans", "logs", "scan-1", "--json"], + output, + process.stderr, + deps, + ); + console.log( + JSON.stringify({ + exitCode, + bytes, + sha256: actual.digest("hex"), + expectedBytes, + expectedSha256: expected.digest("hex"), + maximumStringLength: constants.MAX_STRING_LENGTH, + events: count + 1, + }), + ); + process.exitCode = exitCode; +} catch (error) { + console.error(error); + process.exitCode = 2; +}