diff --git a/CHANGELOG.md b/CHANGELOG.md index 22c91d5..b16c5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ This project uses a Keep a Changelog-style structure. Until a tagged release is - HTTP target observation always runs through the logged, size-bounded network session under the saved run policy. The second retrieval path, reachable only by direct library callers, did not reject content encodings, cap total bytes or keep a request log; it was removed. No CLI command used it. +- Commands that validate a run and then write now confirm every file the validation read through one shared check, `assertValidatedRunStable`, instead of each carrying its own loops. The copies covered registered artifacts and saved evidence but only two also covered the historical resources used to validate older run formats, so the other commands could write after one of those files changed. All commands now share one list, and a test rejects new hand-written copies. + - Removed internal plans, implementation reviews, status notes, and research source copies from the public source tree; retained official source links and attribution. - Reviewed the Digital Agency accessibility policy against the monitored source on 2026-09-25. The 18 additional criterion mappings and routing are unchanged; the catalog source hash, review date and attribution binding now match the reviewed page. diff --git a/release-files.json b/release-files.json index 0ebe903..a7247ab 100644 --- a/release-files.json +++ b/release-files.json @@ -439,6 +439,7 @@ "tests/target-observer.test.mjs", "tests/target-registration.test.mjs", "tests/unified-cli.test.mjs", + "tests/validated-run-stability.test.mjs", "tests/web-capabilities-e2e.test.mjs", "tests/web-capabilities.test.mjs", "tests/web-evidence-adapter.test.mjs", diff --git a/shared/skill/scripts/bind-audit-evidence.mjs b/shared/skill/scripts/bind-audit-evidence.mjs index abd03c7..b499d09 100644 --- a/shared/skill/scripts/bind-audit-evidence.mjs +++ b/shared/skill/scripts/bind-audit-evidence.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; import { createRunEvidenceReference, collectScreeningEvidence } from "./lib/run-evidence.mjs"; function parseArgs(argv) { @@ -55,8 +55,8 @@ export function main(argv = process.argv.slice(2)) { assertStableFile(runSnapshot, "audit run"); assertStableFile(artifactSnapshot, "draft artifact"); assertStableFile(snapshot, "raw evidence"); - for (const { snapshot: registered } of validation.envelopesById.values()) assertStableFile(registered, "registered artifact"); - for (const raw of [...validation.evidenceSnapshots.values(), ...evidence.snapshots.values()]) assertStableFile(raw, "raw evidence"); + assertValidatedRunStable(validation); + for (const raw of evidence.snapshots.values()) assertStableFile(raw, "raw evidence"); writeNewJson(output, artifact); process.stdout.write(`${JSON.stringify({ status: "PASS", output, observation: options.observation, evidence_count: observation.evidence_refs.length })}\n`); } diff --git a/shared/skill/scripts/capture-run-targets.mjs b/shared/skill/scripts/capture-run-targets.mjs index bbd86e1..02d0dce 100644 --- a/shared/skill/scripts/capture-run-targets.mjs +++ b/shared/skill/scripts/capture-run-targets.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; import { observeRunTargetsWithEvidence } from "./lib/run-targets.mjs"; import { afterChangeRun } from "./lib/declared-change.mjs"; import { networkPolicyHash } from "./lib/network-policy.mjs"; @@ -66,8 +66,7 @@ export async function main(argv = process.argv.slice(2)) { const { inventory, networkEvidence } = captured; assertStableFile(runSnapshot, "audit run input"); assertStableFile(specsSnapshot, "target specifications"); - for (const { snapshot } of validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(validation); if (networkOutput) writeNewJson(networkOutput, { schema_version: "1.0.0", kind: "run-network-evidence", run_id: run.run_id, network_policy_sha256: needsNetwork ? networkPolicyHash(run.permissions.network_policy) : null, publication: "private_by_default", observations: networkEvidence }); diff --git a/shared/skill/scripts/compare-audit-evidence.mjs b/shared/skill/scripts/compare-audit-evidence.mjs index 50d946e..2d2bd50 100644 --- a/shared/skill/scripts/compare-audit-evidence.mjs +++ b/shared/skill/scripts/compare-audit-evidence.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; import { compareEvidenceReferences } from "./lib/run-evidence.mjs"; export function main(argv = process.argv.slice(2)) { @@ -28,8 +28,7 @@ export function main(argv = process.argv.slice(2)) { comparisons: compareEvidenceReferences(before.rows, after.rows) }; for (const input of inputs) { assertStableFile(input.snapshot, "comparison run"); - for (const { snapshot } of input.validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const raw of input.validation.evidenceSnapshots.values()) assertStableFile(raw, "raw evidence"); + assertValidatedRunStable(input.validation); } writeNewJson(options["--output"], result); process.stdout.write(`${JSON.stringify({ status: "PASS", output: options["--output"], comparisons: result.comparisons.length })}\n`); diff --git a/shared/skill/scripts/compare-audit-runs.mjs b/shared/skill/scripts/compare-audit-runs.mjs index 120789d..e72cbeb 100644 --- a/shared/skill/scripts/compare-audit-runs.mjs +++ b/shared/skill/scripts/compare-audit-runs.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson, writeNewText } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson, writeNewText } from "./lib/audit-run.mjs"; import { buildRetestDelta, renderRetestDelta } from "./lib/retest-delta.mjs"; function parseOptions(argv) { @@ -45,8 +45,7 @@ export function main(argv = process.argv.slice(2)) { const assertInputsStable = () => { for (const input of [before, after]) { assertStableFile(input.snapshot, "comparison run"); - for (const { snapshot } of input.validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of input.validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(input.validation); } if (mappingSnapshot) assertStableFile(mappingSnapshot, "comparison mapping"); }; diff --git a/shared/skill/scripts/compare-run-targets.mjs b/shared/skill/scripts/compare-run-targets.mjs index 7d51c07..a143a9e 100644 --- a/shared/skill/scripts/compare-run-targets.mjs +++ b/shared/skill/scripts/compare-run-targets.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; import { compareRunTargets } from "./lib/run-targets.mjs"; export function main(argv = process.argv.slice(2)) { @@ -29,8 +29,7 @@ export function main(argv = process.argv.slice(2)) { const comparisons = compareRunTargets(before.run, before.run.target_inventory, after.run, after.run.target_inventory); for (const input of inputs) { assertStableFile(input.snapshot, "comparison run"); - for (const { snapshot } of input.validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of input.validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(input.validation); } writeNewJson(output, { schema_version: "1.0.0", publication: "private_by_default", before_run_id: before.run.run_id, after_run_id: after.run.run_id, comparisons, limitation: "Measured identity differences do not establish accessibility improvement. This comparison does not contact live targets." }); diff --git a/shared/skill/scripts/create-audit-artifact.mjs b/shared/skill/scripts/create-audit-artifact.mjs index 00cefa5..24c1f04 100644 --- a/shared/skill/scripts/create-audit-artifact.mjs +++ b/shared/skill/scripts/create-audit-artifact.mjs @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validateArtifactCandidate, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validatedRunSnapshots, validateArtifactCandidate, writeNewJson } from "./lib/audit-run.mjs"; import { parseAttestationJson } from "./lib/attestation-canonical.mjs"; import { createRunEvidenceReference } from "./lib/run-evidence.mjs"; @@ -34,11 +34,8 @@ export function main(argv = process.argv.slice(2)) { const runFile = path.resolve(options.run), run = read(runFile); const validation = validateAuditRun(run, { runFile }); if (!validation.valid) throw new Error(validation.errors.join("\n")); - for (const { snapshot } of validation.envelopesById.values()) { - parseAttestationJson(snapshot.bytes); - snapshots.push(snapshot); - } - snapshots.push(...validation.evidenceSnapshots.values()); + for (const { snapshot } of validation.envelopesById.values()) parseAttestationJson(snapshot.bytes); + snapshots.push(...validatedRunSnapshots(validation)); let artifact, output; if (action === "init") { if (!types.includes(options.type)) throw new Error(`--type must be one of: ${types.join(", ")}`); diff --git a/shared/skill/scripts/create-human-review-queue.mjs b/shared/skill/scripts/create-human-review-queue.mjs index 4f84ae9..ad68eb6 100644 --- a/shared/skill/scripts/create-human-review-queue.mjs +++ b/shared/skill/scripts/create-human-review-queue.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; import { createHumanReviewQueue, queueContextErrors } from "./lib/human-review-queue.mjs"; export function main(argv = process.argv.slice(2)) { @@ -39,8 +39,7 @@ export function main(argv = process.argv.slice(2)) { const errors = [...schema.errors, ...queueContextErrors(run, new Map([...validation.envelopesById, [artifact.artifact_id, artifact]]), profileIds)]; if (errors.length) throw new Error(errors.join("\n")); assertStableFile(snapshot, "audit run"); - for (const record of validation.envelopesById.values()) assertStableFile(record.snapshot, "registered artifact"); - for (const evidence of validation.evidenceSnapshots.values()) assertStableFile(evidence, "saved evidence"); + assertValidatedRunStable(validation, { evidence: "saved evidence" }); writeNewJson(output, artifact); process.stdout.write(`${JSON.stringify({ status: "candidate", output, items: artifact.payload.items.length, priority: "unprioritized" })}\n`); } diff --git a/shared/skill/scripts/human-review-worksheet.mjs b/shared/skill/scripts/human-review-worksheet.mjs index beaadb9..efbffd4 100644 --- a/shared/skill/scripts/human-review-worksheet.mjs +++ b/shared/skill/scripts/human-review-worksheet.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewBytes, validateHumanReviewCandidate } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, validatedRunSnapshots, writeNewBytes, validateHumanReviewCandidate } from "./lib/audit-run.mjs"; import { parseAttestationJson } from "./lib/attestation-canonical.mjs"; import { worksheetItems, createWorksheetRows, encodeCsv, decodeCsv, encodeMarkdown, decodeMarkdown, reviewFromWorksheet, worksheetLimit } from "./lib/human-review-worksheet.mjs"; import { compareInstants } from "./lib/date-time.mjs"; @@ -43,7 +43,7 @@ export async function runWorksheetCommand(argv) { const relative = path.relative(validation.artifactRoot, output); if (!relative || path.isAbsolute(relative) || relative.split(path.sep).includes("..")) throw new Error("Worksheet and artifact outputs must be within the private artifact root."); assertNewOutputPath(output); - const inputs = [snapshot, ...[...validation.envelopesById.values()].map((entry) => entry.snapshot), ...validation.evidenceSnapshots.values()]; + const inputs = [snapshot, ...validatedRunSnapshots(validation)]; const assertInputs = () => { for (const input of inputs) assertStableFile(input, "worksheet input"); }; if (options.command === "export") { const bytes = options.format === "xlsx" ? await (await import("./lib/human-review-xlsx.mjs")).encodeXlsx(expected) diff --git a/shared/skill/scripts/human-review.mjs b/shared/skill/scripts/human-review.mjs index 82a3453..d1c3df5 100644 --- a/shared/skill/scripts/human-review.mjs +++ b/shared/skill/scripts/human-review.mjs @@ -1,7 +1,7 @@ import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; -import { readStableFile, assertStableFile, writeNewJson, validateAuditRun, loadAuditResources } from "./lib/audit-run.mjs"; +import { readStableFile, assertStableFile, writeNewJson, validateAuditRun, validatedRunSnapshots, loadAuditResources } from "./lib/audit-run.mjs"; import { validateAssessment } from "./validate-assessment.mjs"; import { parseAttestationJson } from "./lib/attestation-canonical.mjs"; import { createAttestationTrust } from "./lib/attestation-verifier.mjs"; @@ -85,8 +85,8 @@ export function runHumanReviewCommand(argv) { const source = validation.envelopesById.get(options["--artifact-id"]); if (!source || source.envelope.artifact_type !== "declared-human-review") throw new Error("--artifact-id must select a registered declared-human-review in the supplied run."); // Every registered artifact must have an unambiguous JSON representation. - for (const entry of validation.envelopesById.values()) { parseAttestationJson(entry.snapshot.bytes); snapshots.push(entry.snapshot); } - snapshots.push(...validation.evidenceSnapshots.values()); + for (const entry of validation.envelopesById.values()) parseAttestationJson(entry.snapshot.bytes); + snapshots.push(...validatedRunSnapshots(validation)); context = humanReviewRunContext({ run, artifact: source.envelope, artifactSha256: source.snapshot.sha256 }); sourceArtifactBytes = source.snapshot.bytes; review = source.envelope.payload; diff --git a/shared/skill/scripts/import-scanner-results.mjs b/shared/skill/scripts/import-scanner-results.mjs index 142dd4c..cd1550d 100644 --- a/shared/skill/scripts/import-scanner-results.mjs +++ b/shared/skill/scripts/import-scanner-results.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; import { buildAxeImport, importedScreeningArtifact } from "./lib/scanner-import.mjs"; import { createRunEvidenceReference, collectScreeningEvidence } from "./lib/run-evidence.mjs"; import { targetSpecification } from "./lib/run-targets.mjs"; @@ -65,8 +65,7 @@ export function main(argv = process.argv.slice(2)) { assertStableFile(runSnapshot, "audit run"); assertStableFile(inputSnapshot, "raw scanner result"); if (configSnapshot) assertStableFile(configSnapshot, "scanner configuration"); - for (const { snapshot } of validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of validation.evidenceSnapshots.values()) assertStableFile(snapshot, "saved evidence"); + assertValidatedRunStable(validation, { evidence: "saved evidence" }); const current = observeLocalTarget(targetSpecification(targetSnapshot), { baseDir: path.dirname(runFile) }).snapshot; if (current.snapshot_id !== targetSnapshot.snapshot_id) throw new Error("Target drift in the saved scanner capture."); writeNewJson(recordFile, record); diff --git a/shared/skill/scripts/lib/audit-run-validation.mjs b/shared/skill/scripts/lib/audit-run-validation.mjs index dbf94f6..53d47e1 100644 --- a/shared/skill/scripts/lib/audit-run-validation.mjs +++ b/shared/skill/scripts/lib/audit-run-validation.mjs @@ -10,7 +10,7 @@ import { validateParticipantBindings, collectParticipantEvidence } from "./parti import { validateDeclaredChangeBindings, collectDeclaredChangeEvidence } from "./declared-change.mjs"; import { collectFixExecutionEvidence } from "./fix-execution-evidence.mjs"; import { targetBindingErrors } from "./run-targets.mjs"; -import { pathKey, parseJsonBytes, readStableFile, resolveInside } from "./safe-file-io.mjs"; +import { pathKey, parseJsonBytes, readStableFile, assertStableFile, resolveInside } from "./safe-file-io.mjs"; import { defaultSkillRoot, loadAuditResources } from "./audit-resources.mjs"; import { auditRunRegistryCompatibility, auditRunEnvelopeCompatibility } from "./orchestration-contracts.mjs"; import { artifactRootFor, validateArtifact, registeredArtifactPath, canonicalPermissions, remediationPermissionError, validateRegisteredArtifactEntries, validateArtifactEnvelopeSemantics, validateHumanQueueBindings, validateScreeningProfileBindings, validateDeclaredHumanBindings, validateRemediationBindings } from "./audit-artifact-validation.mjs"; @@ -224,3 +224,22 @@ export function validateAuditRun(run, { skillRoot = defaultSkillRoot, runFile, r return { valid: errors.length === 0, errors, resources: runResources, artifactRoot, envelopesById, historicalResourceSnapshots, evidenceSnapshots: new Map([...evidence.snapshots, ...contextEvidence.snapshots, ...participantEvidence.snapshots, ...changeEvidence.snapshots, ...fixEvidence.snapshots]) }; } + +// The one list of files validateAuditRun read. A command that writes after validating must confirm each of them still +// holds the validated bytes. Use the two functions below instead of looping over the collections by hand: a collection +// missing from a hand-written loop is a time-of-check to time-of-use gap, and a new collection is added only here. +function* validatedRunInputs(validation) { + for (const snapshot of validation.historicalResourceSnapshots ?? []) yield { kind: "historical", snapshot }; + for (const { snapshot } of validation.envelopesById?.values() ?? []) yield { kind: "artifact", snapshot }; + for (const snapshot of validation.evidenceSnapshots?.values() ?? []) yield { kind: "evidence", snapshot }; +} + +export function validatedRunSnapshots(validation) { + return [...validatedRunInputs(validation)].map(({ snapshot }) => snapshot); +} + +// Labels name the input in the failure message; pass { artifact, evidence, historical } to override the defaults. +export function assertValidatedRunStable(validation, labels = {}) { + const names = { artifact: "registered artifact", evidence: "raw evidence", historical: "historical resource", ...labels }; + for (const { kind, snapshot } of validatedRunInputs(validation)) assertStableFile(snapshot, names[kind]); +} diff --git a/shared/skill/scripts/lib/audit-run.mjs b/shared/skill/scripts/lib/audit-run.mjs index de917a0..4bfd722 100644 --- a/shared/skill/scripts/lib/audit-run.mjs +++ b/shared/skill/scripts/lib/audit-run.mjs @@ -20,13 +20,13 @@ import { pathKey, isWithinPath, hasTraversal, inspectRealComponents, parseJsonBy import { defaultSkillRoot, loadAuditResources } from "./audit-resources.mjs"; import { compareText } from "./text-order.mjs"; import { roleFor, validateArtifact, registeredArtifactPath, canonicalPermissions, remediationPermissionError, validateFixHandoffBinding, validateChangeRecordAuthorizationBinding, assertCurrentOperationalRun } from "./audit-artifact-validation.mjs"; -import { validateAuditRun } from "./audit-run-validation.mjs"; +import { validateAuditRun, assertValidatedRunStable } from "./audit-run-validation.mjs"; export { canonicalJson } from "./canonical-json.mjs"; export { inspectRealComponents, prepareSafeOutputDirectory, assertNewOutputPath, writeNewJson, writeNewText, writeNewBytes, sha256Bytes, sha256File, readStableFile, assertStableFile, resolveInside } from "./safe-file-io.mjs"; export { loadAuditResources, defaultSkillRoot } from "./audit-resources.mjs"; export { validateArtifact, validateArtifactCandidate, validateHumanReviewCandidate } from "./audit-artifact-validation.mjs"; -export { validateAuditRun } from "./audit-run-validation.mjs"; +export { validateAuditRun, assertValidatedRunStable, validatedRunSnapshots } from "./audit-run-validation.mjs"; export { mergeArtifacts } from "./audit-run-merge.mjs"; function normalizePermission(value, aliases, name) { @@ -257,7 +257,6 @@ export function registerArtifact(run, artifact, options = {}) { const nextValidation = validateAuditRun(next, { skillRoot, runFile: options.runFile }); if (!nextValidation.valid) throw new Error(`Registered run is invalid:\n- ${nextValidation.errors.join("\n- ")}`); assertStableFile(snapshot, "artifact file"); - for (const { snapshot: registeredSnapshot } of nextValidation.envelopesById.values()) assertStableFile(registeredSnapshot, "registered artifact"); - for (const evidenceSnapshot of nextValidation.evidenceSnapshots.values()) assertStableFile(evidenceSnapshot, "raw evidence"); + assertValidatedRunStable(nextValidation); return next; } diff --git a/shared/skill/scripts/lib/finding-lifecycle.mjs b/shared/skill/scripts/lib/finding-lifecycle.mjs index 09ac048..f706788 100644 --- a/shared/skill/scripts/lib/finding-lifecycle.mjs +++ b/shared/skill/scripts/lib/finding-lifecycle.mjs @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { isDeepStrictEqual } from "node:util"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, writeNewJson } from "./audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, resolveInside, validateAuditRun, validatedRunSnapshots, writeNewJson } from "./audit-run.mjs"; import { compareInstants, isCalendarDate } from "./date-time.mjs"; import { reviewEntries, resolveHumanReviews, consensusReviewRows, consensusRemediationItems } from "./human-review-consensus.mjs"; import { validateJsonSchema } from "./json-schema.mjs"; @@ -98,8 +98,7 @@ function validateClosure(record, run, validation, runFile) { path.resolve(successorValidation.artifactRoot, state.closure.verification_file), { existing: true }); const evidence = readStableFile(evidencePath, { label: "closure verification evidence" }); if (evidence.sha256 !== state.closure.verification_sha256) throw new Error("Closure verification evidence hash mismatch."); - return [retestSnapshot, evidence, ...[...successorValidation.envelopesById.values()].map((item) => item.snapshot), - ...successorValidation.evidenceSnapshots.values()]; + return [retestSnapshot, evidence, ...validatedRunSnapshots(successorValidation)]; } function validateRecord(record, previous, source, run, validation, runFile) { @@ -252,8 +251,7 @@ export function writeLifecycle(output, record, { run, validation, runFile, input inputSnapshots.push(...validateRecord(record, previous, source, run, validation, runFile)); const stable = () => { for (const snapshot of inputSnapshots) assertStableFile(snapshot, "lifecycle input"); - for (const item of validation.envelopesById.values()) assertStableFile(item.snapshot, "registered artifact"); - for (const snapshot of validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(validation); }; return writeNewJson(target, record, { beforeWrite: stable }); } diff --git a/shared/skill/scripts/lib/network-cli.mjs b/shared/skill/scripts/lib/network-cli.mjs index fca04a9..08bf700 100644 --- a/shared/skill/scripts/lib/network-cli.mjs +++ b/shared/skill/scripts/lib/network-cli.mjs @@ -1,5 +1,5 @@ import path from "node:path"; -import { readStableFile, validateAuditRun, assertStableFile, assertNewOutputPath, writeNewJson } from "./audit-run.mjs"; +import { readStableFile, validateAuditRun, assertStableFile, assertValidatedRunStable, assertNewOutputPath, writeNewJson } from "./audit-run.mjs"; import { assertNetworkPolicy, callerNetworkScope, networkPolicyHash } from "./network-policy.mjs"; export async function withNetworkEvidenceOutput({ run, artifactRoot, output, needsNetwork, inputs = [], otherOutputs = [] }, operation) { @@ -44,8 +44,7 @@ export function prepareNetworkCapture(options, outputs = []) { options.networkCaller = caller; return { save(log) { assertStableFile(snapshot, "network run input"); - for (const record of validation.envelopesById.values()) assertStableFile(record.snapshot, "registered artifact"); - for (const raw of validation.evidenceSnapshots.values()) assertStableFile(raw, "raw evidence"); + assertValidatedRunStable(validation); writeNewJson(output, log); } }; } diff --git a/shared/skill/scripts/merge-audit-artifacts.mjs b/shared/skill/scripts/merge-audit-artifacts.mjs index 917ca4d..fcc23fc 100644 --- a/shared/skill/scripts/merge-audit-artifacts.mjs +++ b/shared/skill/scripts/merge-audit-artifacts.mjs @@ -4,6 +4,7 @@ import { pathToFileURL } from "node:url"; import { assertStableFile, + assertValidatedRunStable, assertNewOutputPath, loadAuditResources, mergeArtifacts, @@ -91,8 +92,7 @@ export function main(argv = process.argv.slice(2)) { assertStableFile(runSnapshot, "audit run input"); assertStableFile(assessmentSnapshot, "assessment input"); for (const snapshot of artifactSnapshots) assertStableFile(snapshot, "merge artifact"); - for (const { snapshot } of runValidation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of runValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(runValidation); writeNewJson(output, merged, { beforeWrite() { for (const snapshot of [...trustInput.snapshots, ...reviewInputs.map((input) => input.snapshot)]) assertStableFile(snapshot, "review verification input"); } }); diff --git a/shared/skill/scripts/register-audit-artifact.mjs b/shared/skill/scripts/register-audit-artifact.mjs index 77b2c35..4d2d6d9 100644 --- a/shared/skill/scripts/register-audit-artifact.mjs +++ b/shared/skill/scripts/register-audit-artifact.mjs @@ -5,6 +5,7 @@ import { withNetworkEvidenceOutput } from "./lib/network-cli.mjs"; import { assertStableFile, + assertValidatedRunStable, assertNewOutputPath, readStableFile, registerArtifactChecked, @@ -65,8 +66,7 @@ export async function main(argv = process.argv.slice(2)) { assertStableFile(artifactSnapshot, "artifact input"); const finalValidation = validateAuditRun(next, { runFile }); if (!finalValidation.valid) throw new Error(`Registered audit run failed final validation:\n- ${finalValidation.errors.join("\n- ")}`); - for (const { snapshot } of finalValidation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of finalValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(finalValidation); writeNewJson(output, next); process.stdout.write(`${JSON.stringify({ status: "PASS", run_id: next.run_id, run_status: next.status, artifact_id: artifact.artifact_id, output })}\n`); } diff --git a/shared/skill/scripts/render-audit-report.mjs b/shared/skill/scripts/render-audit-report.mjs index 0225c11..41d723e 100644 --- a/shared/skill/scripts/render-audit-report.mjs +++ b/shared/skill/scripts/render-audit-report.mjs @@ -7,7 +7,7 @@ import { reviewDetailLines } from "./lib/review-details.mjs"; import path from "node:path"; import process from "node:process"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { assertStableFile, readStableFile, validateAuditRun, writeNewText } from "./lib/audit-run.mjs"; +import { assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewText } from "./lib/audit-run.mjs"; import { validateAssessment } from "./validate-assessment.mjs"; import { reviewerAssuranceText, reviewerVerificationOptions, displayedEvidenceLevel } from "./lib/assessment-provenance.mjs"; import { loadReviewTrust } from "./lib/review-trust-input.mjs"; @@ -457,13 +457,11 @@ function main() { resources: runValidation.resources, trust: trustInput.trust })); - const artifactSnapshots = [...runValidation.envelopesById.values()].map((record) => record.snapshot).filter(Boolean); const output = writeNewText(path.resolve(options.output), report, { beforeWrite() { assertStableFile(runSnapshot, "audit run"); assertStableFile(assessmentSnapshot, "merged assessment"); - for (const snapshot of artifactSnapshots) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of runValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(runValidation); for (const snapshot of trustInput.snapshots) assertStableFile(snapshot, "external reviewer trust policy"); } }); diff --git a/shared/skill/scripts/render-report.mjs b/shared/skill/scripts/render-report.mjs index c9b9de6..c80b872 100755 --- a/shared/skill/scripts/render-report.mjs +++ b/shared/skill/scripts/render-report.mjs @@ -8,6 +8,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { assertNewOutputPath, assertStableFile, + assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson, @@ -323,15 +324,10 @@ function renderRunBacked(options) { rendered.report = decorate(rendered.report); if (rendered.appendix) rendered.appendix = decorate(rendered.appendix); } - const artifactSnapshots = [...runValidation.envelopesById.values()] - .map((record) => record.snapshot) - .filter(Boolean); const assertInputsStable = () => { assertStableFile(runSnapshot, "audit run"); assertStableFile(assessmentSnapshot, "run-backed assessment"); - for (const snapshot of artifactSnapshots) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of runValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); - for (const snapshot of runValidation.historicalResourceSnapshots ?? []) assertStableFile(snapshot, "historical resource"); + assertValidatedRunStable(runValidation); for (const snapshot of lifecycleSnapshots) assertStableFile(snapshot, "lifecycle input"); for (const snapshot of trustInput.snapshots) assertStableFile(snapshot, "external reviewer trust policy"); }; diff --git a/shared/skill/scripts/show-audit-status.mjs b/shared/skill/scripts/show-audit-status.mjs index f7d5bd8..3698c04 100644 --- a/shared/skill/scripts/show-audit-status.mjs +++ b/shared/skill/scripts/show-audit-status.mjs @@ -11,7 +11,7 @@ import { interactionScopeSummary } from "./lib/interaction-policy.mjs"; import { findingRelations, findingRetestSummary, remediationPlanItems } from "./lib/run-findings.mjs"; import { lifecycleSummary, loadLifecycle } from "./lib/finding-lifecycle.mjs"; import { reviewEntries, resolveHumanReviews, consensusReviewRows, consensusRemediationItems } from "./lib/human-review-consensus.mjs"; -import { assertStableFile, defaultSkillRoot, mergeArtifacts, readStableFile, validateAuditRun } from "./lib/audit-run.mjs"; +import { assertStableFile, assertValidatedRunStable, defaultSkillRoot, mergeArtifacts, readStableFile, validateAuditRun } from "./lib/audit-run.mjs"; function parse(snapshot) { return JSON.parse(snapshot.bytes.toString("utf8").replace(/^\uFEFF/u, "")); @@ -52,8 +52,7 @@ function discoverSuccessors(run, runFile, artifactRoot, skillRoot) { continue; } assertStableFile(snapshot, "sibling run candidate"); - for (const record of validation.envelopesById.values()) assertStableFile(record.snapshot, "sibling registered artifact"); - for (const evidence of validation.evidenceSnapshots.values()) assertStableFile(evidence, "sibling raw evidence"); + assertValidatedRunStable(validation, { artifact: "sibling registered artifact", evidence: "sibling raw evidence" }); if (retest || (validation.artifactRoot === artifactRoot && extendsRun(candidate, run))) { candidates.push({ file: entry.name, run_id: candidate.run_id, state: candidate.status, kind: retest ? "retest" : "registration" }); } else if (sameRun && !isDeepStrictEqual(candidate, run) && !extendsRun(run, candidate)) { @@ -161,8 +160,7 @@ export function auditStatus(runFile, { skillRoot = defaultSkillRoot, retestOf, l || !isDeepStrictEqual(run.scope, prior.scope) || !isDeepStrictEqual(run.inspection_request, prior.inspection_request)) throw new Error("Retest comparison requires the linked predecessor and the same target/profile/scope at a new version."); priorEnvelopes = [...priorValidation.envelopesById.values()].map((record) => record.envelope); comparison = true; assertStableFile(priorSnapshot, "retest predecessor"); - for (const record of priorValidation.envelopesById.values()) assertStableFile(record.snapshot, "predecessor artifact"); - for (const evidence of priorValidation.evidenceSnapshots?.values() ?? []) assertStableFile(evidence, "predecessor evidence"); + assertValidatedRunStable(priorValidation, { artifact: "predecessor artifact", evidence: "predecessor evidence", historical: "predecessor historical resource" }); } const resolve = (records) => resolveHumanReviews(reviewEntries(records.filter((item) => item.artifact_type === "declared-human-review") .map((item) => ({ payload: item.payload, artifact_id: item.artifact_id })))); @@ -177,8 +175,7 @@ export function auditStatus(runFile, { skillRoot = defaultSkillRoot, retestOf, l })); } assertStableFile(snapshot, "audit run"); - for (const record of validation.envelopesById?.values() ?? []) assertStableFile(record.snapshot, "registered artifact"); - for (const evidence of validation.evidenceSnapshots?.values() ?? []) assertStableFile(evidence, "raw evidence"); + assertValidatedRunStable(validation); return result; } diff --git a/shared/skill/scripts/validate-assessment.mjs b/shared/skill/scripts/validate-assessment.mjs index 43e2a91..c10ab98 100644 --- a/shared/skill/scripts/validate-assessment.mjs +++ b/shared/skill/scripts/validate-assessment.mjs @@ -533,7 +533,7 @@ if (isMain) { } if (!positional.length || positional.length > 5) throw new Error("Usage: node validate-assessment.mjs [registry.json] [schema.json] [catalog.json] [methods.json] [--run run.json] [--trust-policy policy.json --trust-policy-sha256 hash]"); const { loadReviewTrust, readReviewJson } = await import("./lib/review-trust-input.mjs"); - const { assertStableFile, validateAuditRun } = await import("./lib/audit-run.mjs"); + const { assertStableFile, validateAuditRun, validatedRunSnapshots } = await import("./lib/audit-run.mjs"); const { reviewerVerificationOptions } = await import("./lib/assessment-provenance.mjs"); const input = readReviewJson(positional[0], "assessment"), trustInput = loadReviewTrust(options); const snapshots = [input.snapshot, ...trustInput.snapshots]; @@ -543,7 +543,7 @@ if (isMain) { const runValidation = validateAuditRun(runInput.value, { runFile: runInput.snapshot.path }); if (!runValidation.valid) throw new Error(`Audit run validation failed: ${runValidation.errors.join("; ")}`); reviewOptions = reviewerVerificationOptions({ run: runInput.value, envelopesById: runValidation.envelopesById, trust: trustInput.trust }); - snapshots.push(runInput.snapshot, ...[...runValidation.envelopesById.values()].map((item) => item.snapshot), ...runValidation.evidenceSnapshots.values()); + snapshots.push(runInput.snapshot, ...validatedRunSnapshots(runValidation)); } const references = ["standards-registry.json", "assessment-record.schema.json", "criteria-catalog.json", "web-audit-methods.json"] .map((name, index) => readJson(positional[index + 1] ?? fileURLToPath(new URL(`../references/${name}`, import.meta.url)))); diff --git a/shared/skill/scripts/validate-audit-run.mjs b/shared/skill/scripts/validate-audit-run.mjs index dfc5890..dbd5fc4 100644 --- a/shared/skill/scripts/validate-audit-run.mjs +++ b/shared/skill/scripts/validate-audit-run.mjs @@ -2,7 +2,7 @@ import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; function parseArgs(argv) { const options = {}; @@ -32,9 +32,7 @@ export function main(argv = process.argv.slice(2)) { const snapshot = readStableFile(input, { label: "audit run input" }); const result = validateAuditRun(parseSnapshot(snapshot), { runFile: input, historicalResourceRoot: options.historicalResources }); assertStableFile(snapshot, "audit run input"); - for (const historical of result.historicalResourceSnapshots ?? []) assertStableFile(historical, "historical resource"); - for (const { snapshot: artifact } of result.envelopesById?.values() ?? []) assertStableFile(artifact, "registered artifact"); - for (const evidence of result.evidenceSnapshots?.values() ?? []) assertStableFile(evidence, "raw evidence"); + assertValidatedRunStable(result); writeNewJson(output, { valid: result.valid, errors: result.errors }); process.stdout.write(`${JSON.stringify({ status: result.valid ? "PASS" : "FAIL", input, output, valid: result.valid })}\n`); if (!result.valid) { diff --git a/tests/validated-run-stability.test.mjs b/tests/validated-run-stability.test.mjs new file mode 100644 index 0000000..8e40c6f --- /dev/null +++ b/tests/validated-run-stability.test.mjs @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { assertValidatedRunStable, validatedRunSnapshots } from "../codex/skills/information-accessibility-practice/scripts/lib/audit-run.mjs"; +import { readStableFile } from "../codex/skills/information-accessibility-practice/scripts/lib/safe-file-io.mjs"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +// A stand-in for what validateAuditRun returns: one registered artifact, one saved evidence file, one historical resource. +function fixture(t) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "a11y-validated-run-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const files = Object.fromEntries(["artifact", "evidence", "historical"].map((name) => { + const file = path.join(directory, `${name}.json`); + fs.writeFileSync(file, JSON.stringify({ name })); + return [name, file]; + })); + const validation = { + valid: true, + envelopesById: new Map([["ARTIFACT-1", { envelope: {}, snapshot: readStableFile(files.artifact) }]]), + evidenceSnapshots: new Map([["evidence.json", readStableFile(files.evidence)]]), + historicalResourceSnapshots: [readStableFile(files.historical)] + }; + return { files, validation }; +} + +test("a validated run passes while nothing it read has changed", (t) => { + const { validation } = fixture(t); + assert.doesNotThrow(() => assertValidatedRunStable(validation)); + assert.equal(validatedRunSnapshots(validation).length, 3); +}); + +for (const [name, label] of [["artifact", "registered artifact"], ["evidence", "raw evidence"], ["historical", "historical resource"]]) { + test(`a change to the ${name} input after validation is refused with its label`, (t) => { + const { files, validation } = fixture(t); + fs.writeFileSync(files[name], JSON.stringify({ name, edited: true })); + assert.throws(() => assertValidatedRunStable(validation), new RegExp(`${label} changed before commit`, "u")); + }); +} + +test("callers can rename the inputs, and a validation without snapshots is tolerated", (t) => { + const { files, validation } = fixture(t); + fs.writeFileSync(files.evidence, "{}"); + assert.throws(() => assertValidatedRunStable(validation, { evidence: "saved evidence" }), /saved evidence changed before commit/u); + assert.doesNotThrow(() => assertValidatedRunStable({ valid: false, errors: ["rejected"] })); + assert.deepEqual(validatedRunSnapshots({ valid: false, errors: ["rejected"] }), []); +}); + +function sourceFiles(directory) { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(directory, entry.name); + if (entry.isDirectory()) return sourceFiles(full); + return entry.name.endsWith(".mjs") ? [full] : []; + }); +} + +test("no command re-implements the post-validation re-check by hand", () => { + const helper = path.join(root, "shared/skill/scripts/lib/audit-run-validation.mjs"); + const offenders = sourceFiles(path.join(root, "shared/skill/scripts")).filter((file) => file !== helper).flatMap((file) => + fs.readFileSync(file, "utf8").split(/\r?\n/u).flatMap((line, index) => + /assertStableFile\(/u.test(line) && /\b(?:envelopesById|evidenceSnapshots|historicalResourceSnapshots)\b/u.test(line) + ? [`${path.relative(root, file).split(path.sep).join("/")}:${index + 1}`] : [])); + assert.deepEqual(offenders, [], "use assertValidatedRunStable or validatedRunSnapshots from lib/audit-run.mjs"); +});