From 7ef351a992df83f766667bb07eb5e594bec6b0b2 Mon Sep 17 00:00:00 2001 From: Nagi-Inaba <221035391+Nagi-Inaba@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:44:13 +0900 Subject: [PATCH] refactor: re-check validated run inputs through one shared helper A command that validates a run and then writes must confirm that every file the validation read still holds the validated bytes, otherwise the file can be swapped between the check and the write. validateAuditRun returns three such collections: registered artifacts, saved evidence and the historical resources used to validate older run formats. Each command carried its own copy of the loops, and the copies disagreed: only render-report.mjs and validate-audit-run.mjs also covered the historical resources, so every other command could write after one of those files changed. lib/audit-run-validation.mjs now owns the single list of those inputs and exports - assertValidatedRunStable(validation, labels) to re-check them, and - validatedRunSnapshots(validation) for commands that collect snapshots and re-check them in a write hook. Both are re-exported by lib/audit-run.mjs. The 18 hand-written re-check blocks and 5 hand-written collections in 20 files are replaced one for one, keeping the labels that appear in failure messages (saved evidence, sibling registered artifact, predecessor evidence and so on). A command that renders or writes now also covers the historical resources, which closes the gap above; the same helper accepts the shapes of an invalid validation so status reporting is unaffected. tests/validated-run-stability.test.mjs covers each of the three collections refusing a change with its label, custom labels, tolerance of a validation without snapshots, and a guard that fails when a script reintroduces a hand-written loop over validateAuditRun's collections. The hand-written argument parsers, the BOM-stripping JSON read idiom and the usage text are intentionally left as they are: the parsers check their arguments in at least six different orders, so sharing one would change some commands' error precedence or need an ordering parameter, and none of them is a safety property. Verified: verify-all passes on Node.js 22.19.0 (Windows), 92 test files. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + release-files.json | 1 + shared/skill/scripts/bind-audit-evidence.mjs | 6 +- shared/skill/scripts/capture-run-targets.mjs | 5 +- .../skill/scripts/compare-audit-evidence.mjs | 5 +- shared/skill/scripts/compare-audit-runs.mjs | 5 +- shared/skill/scripts/compare-run-targets.mjs | 5 +- .../skill/scripts/create-audit-artifact.mjs | 9 +-- .../scripts/create-human-review-queue.mjs | 5 +- .../skill/scripts/human-review-worksheet.mjs | 4 +- shared/skill/scripts/human-review.mjs | 6 +- .../skill/scripts/import-scanner-results.mjs | 5 +- .../scripts/lib/audit-run-validation.mjs | 21 +++++- shared/skill/scripts/lib/audit-run.mjs | 7 +- .../skill/scripts/lib/finding-lifecycle.mjs | 8 +-- shared/skill/scripts/lib/network-cli.mjs | 5 +- .../skill/scripts/merge-audit-artifacts.mjs | 4 +- .../skill/scripts/register-audit-artifact.mjs | 4 +- shared/skill/scripts/render-audit-report.mjs | 6 +- shared/skill/scripts/render-report.mjs | 8 +-- shared/skill/scripts/show-audit-status.mjs | 11 ++- shared/skill/scripts/validate-assessment.mjs | 4 +- shared/skill/scripts/validate-audit-run.mjs | 6 +- tests/validated-run-stability.test.mjs | 67 +++++++++++++++++++ 24 files changed, 137 insertions(+), 72 deletions(-) create mode 100644 tests/validated-run-stability.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4be5b9bb..56636f9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ This project uses a Keep a Changelog-style structure. Until a tagged release is ### Changed +- Commands that validate a run and then write now confirm every file the validation read through one shared check, `assertValidatedRunStable`, instead of each carrying its own loops. The copies covered registered artifacts and saved evidence but only two also covered the historical resources used to validate older run formats, so the other commands could write after one of those files changed. All commands now share one list, and a test rejects new hand-written copies. + - Removed internal plans, implementation reviews, status notes, and research source copies from the public source tree; retained official source links and attribution. - Reviewed the Digital Agency accessibility policy against the monitored source on 2026-09-25. The 18 additional criterion mappings and routing are unchanged; the catalog source hash, review date and attribution binding now match the reviewed page. diff --git a/release-files.json b/release-files.json index 4581ed50..9afa36c4 100644 --- a/release-files.json +++ b/release-files.json @@ -438,6 +438,7 @@ "tests/target-observer.test.mjs", "tests/target-registration.test.mjs", "tests/unified-cli.test.mjs", + "tests/validated-run-stability.test.mjs", "tests/web-capabilities-e2e.test.mjs", "tests/web-capabilities.test.mjs", "tests/web-evidence-adapter.test.mjs", diff --git a/shared/skill/scripts/bind-audit-evidence.mjs b/shared/skill/scripts/bind-audit-evidence.mjs index abd03c7d..b499d09c 100644 --- a/shared/skill/scripts/bind-audit-evidence.mjs +++ b/shared/skill/scripts/bind-audit-evidence.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; import { createRunEvidenceReference, collectScreeningEvidence } from "./lib/run-evidence.mjs"; function parseArgs(argv) { @@ -55,8 +55,8 @@ export function main(argv = process.argv.slice(2)) { assertStableFile(runSnapshot, "audit run"); assertStableFile(artifactSnapshot, "draft artifact"); assertStableFile(snapshot, "raw evidence"); - for (const { snapshot: registered } of validation.envelopesById.values()) assertStableFile(registered, "registered artifact"); - for (const raw of [...validation.evidenceSnapshots.values(), ...evidence.snapshots.values()]) assertStableFile(raw, "raw evidence"); + assertValidatedRunStable(validation); + for (const raw of evidence.snapshots.values()) assertStableFile(raw, "raw evidence"); writeNewJson(output, artifact); process.stdout.write(`${JSON.stringify({ status: "PASS", output, observation: options.observation, evidence_count: observation.evidence_refs.length })}\n`); } diff --git a/shared/skill/scripts/capture-run-targets.mjs b/shared/skill/scripts/capture-run-targets.mjs index bbd86e1e..02d0dce7 100644 --- a/shared/skill/scripts/capture-run-targets.mjs +++ b/shared/skill/scripts/capture-run-targets.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; import { observeRunTargetsWithEvidence } from "./lib/run-targets.mjs"; import { afterChangeRun } from "./lib/declared-change.mjs"; import { networkPolicyHash } from "./lib/network-policy.mjs"; @@ -66,8 +66,7 @@ export async function main(argv = process.argv.slice(2)) { const { inventory, networkEvidence } = captured; assertStableFile(runSnapshot, "audit run input"); assertStableFile(specsSnapshot, "target specifications"); - for (const { snapshot } of validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(validation); if (networkOutput) writeNewJson(networkOutput, { schema_version: "1.0.0", kind: "run-network-evidence", run_id: run.run_id, network_policy_sha256: needsNetwork ? networkPolicyHash(run.permissions.network_policy) : null, publication: "private_by_default", observations: networkEvidence }); diff --git a/shared/skill/scripts/compare-audit-evidence.mjs b/shared/skill/scripts/compare-audit-evidence.mjs index 50d946ed..2d2bd504 100644 --- a/shared/skill/scripts/compare-audit-evidence.mjs +++ b/shared/skill/scripts/compare-audit-evidence.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; import { compareEvidenceReferences } from "./lib/run-evidence.mjs"; export function main(argv = process.argv.slice(2)) { @@ -28,8 +28,7 @@ export function main(argv = process.argv.slice(2)) { comparisons: compareEvidenceReferences(before.rows, after.rows) }; for (const input of inputs) { assertStableFile(input.snapshot, "comparison run"); - for (const { snapshot } of input.validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const raw of input.validation.evidenceSnapshots.values()) assertStableFile(raw, "raw evidence"); + assertValidatedRunStable(input.validation); } writeNewJson(options["--output"], result); process.stdout.write(`${JSON.stringify({ status: "PASS", output: options["--output"], comparisons: result.comparisons.length })}\n`); diff --git a/shared/skill/scripts/compare-audit-runs.mjs b/shared/skill/scripts/compare-audit-runs.mjs index 120789da..e72cbebe 100644 --- a/shared/skill/scripts/compare-audit-runs.mjs +++ b/shared/skill/scripts/compare-audit-runs.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson, writeNewText } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson, writeNewText } from "./lib/audit-run.mjs"; import { buildRetestDelta, renderRetestDelta } from "./lib/retest-delta.mjs"; function parseOptions(argv) { @@ -45,8 +45,7 @@ export function main(argv = process.argv.slice(2)) { const assertInputsStable = () => { for (const input of [before, after]) { assertStableFile(input.snapshot, "comparison run"); - for (const { snapshot } of input.validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of input.validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(input.validation); } if (mappingSnapshot) assertStableFile(mappingSnapshot, "comparison mapping"); }; diff --git a/shared/skill/scripts/compare-run-targets.mjs b/shared/skill/scripts/compare-run-targets.mjs index 7d51c07b..a143a9e7 100644 --- a/shared/skill/scripts/compare-run-targets.mjs +++ b/shared/skill/scripts/compare-run-targets.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; import { compareRunTargets } from "./lib/run-targets.mjs"; export function main(argv = process.argv.slice(2)) { @@ -29,8 +29,7 @@ export function main(argv = process.argv.slice(2)) { const comparisons = compareRunTargets(before.run, before.run.target_inventory, after.run, after.run.target_inventory); for (const input of inputs) { assertStableFile(input.snapshot, "comparison run"); - for (const { snapshot } of input.validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of input.validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(input.validation); } writeNewJson(output, { schema_version: "1.0.0", publication: "private_by_default", before_run_id: before.run.run_id, after_run_id: after.run.run_id, comparisons, limitation: "Measured identity differences do not establish accessibility improvement. This comparison does not contact live targets." }); diff --git a/shared/skill/scripts/create-audit-artifact.mjs b/shared/skill/scripts/create-audit-artifact.mjs index 00cefa50..24c1f045 100644 --- a/shared/skill/scripts/create-audit-artifact.mjs +++ b/shared/skill/scripts/create-audit-artifact.mjs @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validateArtifactCandidate, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validatedRunSnapshots, validateArtifactCandidate, writeNewJson } from "./lib/audit-run.mjs"; import { parseAttestationJson } from "./lib/attestation-canonical.mjs"; import { createRunEvidenceReference } from "./lib/run-evidence.mjs"; @@ -34,11 +34,8 @@ export function main(argv = process.argv.slice(2)) { const runFile = path.resolve(options.run), run = read(runFile); const validation = validateAuditRun(run, { runFile }); if (!validation.valid) throw new Error(validation.errors.join("\n")); - for (const { snapshot } of validation.envelopesById.values()) { - parseAttestationJson(snapshot.bytes); - snapshots.push(snapshot); - } - snapshots.push(...validation.evidenceSnapshots.values()); + for (const { snapshot } of validation.envelopesById.values()) parseAttestationJson(snapshot.bytes); + snapshots.push(...validatedRunSnapshots(validation)); let artifact, output; if (action === "init") { if (!types.includes(options.type)) throw new Error(`--type must be one of: ${types.join(", ")}`); diff --git a/shared/skill/scripts/create-human-review-queue.mjs b/shared/skill/scripts/create-human-review-queue.mjs index 4f84ae93..ad68eb65 100644 --- a/shared/skill/scripts/create-human-review-queue.mjs +++ b/shared/skill/scripts/create-human-review-queue.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; import { createHumanReviewQueue, queueContextErrors } from "./lib/human-review-queue.mjs"; export function main(argv = process.argv.slice(2)) { @@ -39,8 +39,7 @@ export function main(argv = process.argv.slice(2)) { const errors = [...schema.errors, ...queueContextErrors(run, new Map([...validation.envelopesById, [artifact.artifact_id, artifact]]), profileIds)]; if (errors.length) throw new Error(errors.join("\n")); assertStableFile(snapshot, "audit run"); - for (const record of validation.envelopesById.values()) assertStableFile(record.snapshot, "registered artifact"); - for (const evidence of validation.evidenceSnapshots.values()) assertStableFile(evidence, "saved evidence"); + assertValidatedRunStable(validation, { evidence: "saved evidence" }); writeNewJson(output, artifact); process.stdout.write(`${JSON.stringify({ status: "candidate", output, items: artifact.payload.items.length, priority: "unprioritized" })}\n`); } diff --git a/shared/skill/scripts/human-review-worksheet.mjs b/shared/skill/scripts/human-review-worksheet.mjs index beaadb91..efbffd4a 100644 --- a/shared/skill/scripts/human-review-worksheet.mjs +++ b/shared/skill/scripts/human-review-worksheet.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewBytes, validateHumanReviewCandidate } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, validatedRunSnapshots, writeNewBytes, validateHumanReviewCandidate } from "./lib/audit-run.mjs"; import { parseAttestationJson } from "./lib/attestation-canonical.mjs"; import { worksheetItems, createWorksheetRows, encodeCsv, decodeCsv, encodeMarkdown, decodeMarkdown, reviewFromWorksheet, worksheetLimit } from "./lib/human-review-worksheet.mjs"; import { compareInstants } from "./lib/date-time.mjs"; @@ -43,7 +43,7 @@ export async function runWorksheetCommand(argv) { const relative = path.relative(validation.artifactRoot, output); if (!relative || path.isAbsolute(relative) || relative.split(path.sep).includes("..")) throw new Error("Worksheet and artifact outputs must be within the private artifact root."); assertNewOutputPath(output); - const inputs = [snapshot, ...[...validation.envelopesById.values()].map((entry) => entry.snapshot), ...validation.evidenceSnapshots.values()]; + const inputs = [snapshot, ...validatedRunSnapshots(validation)]; const assertInputs = () => { for (const input of inputs) assertStableFile(input, "worksheet input"); }; if (options.command === "export") { const bytes = options.format === "xlsx" ? await (await import("./lib/human-review-xlsx.mjs")).encodeXlsx(expected) diff --git a/shared/skill/scripts/human-review.mjs b/shared/skill/scripts/human-review.mjs index 82a3453a..d1c3df5c 100644 --- a/shared/skill/scripts/human-review.mjs +++ b/shared/skill/scripts/human-review.mjs @@ -1,7 +1,7 @@ import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; -import { readStableFile, assertStableFile, writeNewJson, validateAuditRun, loadAuditResources } from "./lib/audit-run.mjs"; +import { readStableFile, assertStableFile, writeNewJson, validateAuditRun, validatedRunSnapshots, loadAuditResources } from "./lib/audit-run.mjs"; import { validateAssessment } from "./validate-assessment.mjs"; import { parseAttestationJson } from "./lib/attestation-canonical.mjs"; import { createAttestationTrust } from "./lib/attestation-verifier.mjs"; @@ -85,8 +85,8 @@ export function runHumanReviewCommand(argv) { const source = validation.envelopesById.get(options["--artifact-id"]); if (!source || source.envelope.artifact_type !== "declared-human-review") throw new Error("--artifact-id must select a registered declared-human-review in the supplied run."); // Every registered artifact must have an unambiguous JSON representation. - for (const entry of validation.envelopesById.values()) { parseAttestationJson(entry.snapshot.bytes); snapshots.push(entry.snapshot); } - snapshots.push(...validation.evidenceSnapshots.values()); + for (const entry of validation.envelopesById.values()) parseAttestationJson(entry.snapshot.bytes); + snapshots.push(...validatedRunSnapshots(validation)); context = humanReviewRunContext({ run, artifact: source.envelope, artifactSha256: source.snapshot.sha256 }); sourceArtifactBytes = source.snapshot.bytes; review = source.envelope.payload; diff --git a/shared/skill/scripts/import-scanner-results.mjs b/shared/skill/scripts/import-scanner-results.mjs index 142dd4c6..cd1550d3 100644 --- a/shared/skill/scripts/import-scanner-results.mjs +++ b/shared/skill/scripts/import-scanner-results.mjs @@ -1,6 +1,6 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, resolveInside, validateAuditRun, validateArtifact, writeNewJson } from "./lib/audit-run.mjs"; import { buildAxeImport, importedScreeningArtifact } from "./lib/scanner-import.mjs"; import { createRunEvidenceReference, collectScreeningEvidence } from "./lib/run-evidence.mjs"; import { targetSpecification } from "./lib/run-targets.mjs"; @@ -65,8 +65,7 @@ export function main(argv = process.argv.slice(2)) { assertStableFile(runSnapshot, "audit run"); assertStableFile(inputSnapshot, "raw scanner result"); if (configSnapshot) assertStableFile(configSnapshot, "scanner configuration"); - for (const { snapshot } of validation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of validation.evidenceSnapshots.values()) assertStableFile(snapshot, "saved evidence"); + assertValidatedRunStable(validation, { evidence: "saved evidence" }); const current = observeLocalTarget(targetSpecification(targetSnapshot), { baseDir: path.dirname(runFile) }).snapshot; if (current.snapshot_id !== targetSnapshot.snapshot_id) throw new Error("Target drift in the saved scanner capture."); writeNewJson(recordFile, record); diff --git a/shared/skill/scripts/lib/audit-run-validation.mjs b/shared/skill/scripts/lib/audit-run-validation.mjs index dbf94f6f..53d47e19 100644 --- a/shared/skill/scripts/lib/audit-run-validation.mjs +++ b/shared/skill/scripts/lib/audit-run-validation.mjs @@ -10,7 +10,7 @@ import { validateParticipantBindings, collectParticipantEvidence } from "./parti import { validateDeclaredChangeBindings, collectDeclaredChangeEvidence } from "./declared-change.mjs"; import { collectFixExecutionEvidence } from "./fix-execution-evidence.mjs"; import { targetBindingErrors } from "./run-targets.mjs"; -import { pathKey, parseJsonBytes, readStableFile, resolveInside } from "./safe-file-io.mjs"; +import { pathKey, parseJsonBytes, readStableFile, assertStableFile, resolveInside } from "./safe-file-io.mjs"; import { defaultSkillRoot, loadAuditResources } from "./audit-resources.mjs"; import { auditRunRegistryCompatibility, auditRunEnvelopeCompatibility } from "./orchestration-contracts.mjs"; import { artifactRootFor, validateArtifact, registeredArtifactPath, canonicalPermissions, remediationPermissionError, validateRegisteredArtifactEntries, validateArtifactEnvelopeSemantics, validateHumanQueueBindings, validateScreeningProfileBindings, validateDeclaredHumanBindings, validateRemediationBindings } from "./audit-artifact-validation.mjs"; @@ -224,3 +224,22 @@ export function validateAuditRun(run, { skillRoot = defaultSkillRoot, runFile, r return { valid: errors.length === 0, errors, resources: runResources, artifactRoot, envelopesById, historicalResourceSnapshots, evidenceSnapshots: new Map([...evidence.snapshots, ...contextEvidence.snapshots, ...participantEvidence.snapshots, ...changeEvidence.snapshots, ...fixEvidence.snapshots]) }; } + +// The one list of files validateAuditRun read. A command that writes after validating must confirm each of them still +// holds the validated bytes. Use the two functions below instead of looping over the collections by hand: a collection +// missing from a hand-written loop is a time-of-check to time-of-use gap, and a new collection is added only here. +function* validatedRunInputs(validation) { + for (const snapshot of validation.historicalResourceSnapshots ?? []) yield { kind: "historical", snapshot }; + for (const { snapshot } of validation.envelopesById?.values() ?? []) yield { kind: "artifact", snapshot }; + for (const snapshot of validation.evidenceSnapshots?.values() ?? []) yield { kind: "evidence", snapshot }; +} + +export function validatedRunSnapshots(validation) { + return [...validatedRunInputs(validation)].map(({ snapshot }) => snapshot); +} + +// Labels name the input in the failure message; pass { artifact, evidence, historical } to override the defaults. +export function assertValidatedRunStable(validation, labels = {}) { + const names = { artifact: "registered artifact", evidence: "raw evidence", historical: "historical resource", ...labels }; + for (const { kind, snapshot } of validatedRunInputs(validation)) assertStableFile(snapshot, names[kind]); +} diff --git a/shared/skill/scripts/lib/audit-run.mjs b/shared/skill/scripts/lib/audit-run.mjs index de917a04..4bfd7224 100644 --- a/shared/skill/scripts/lib/audit-run.mjs +++ b/shared/skill/scripts/lib/audit-run.mjs @@ -20,13 +20,13 @@ import { pathKey, isWithinPath, hasTraversal, inspectRealComponents, parseJsonBy import { defaultSkillRoot, loadAuditResources } from "./audit-resources.mjs"; import { compareText } from "./text-order.mjs"; import { roleFor, validateArtifact, registeredArtifactPath, canonicalPermissions, remediationPermissionError, validateFixHandoffBinding, validateChangeRecordAuthorizationBinding, assertCurrentOperationalRun } from "./audit-artifact-validation.mjs"; -import { validateAuditRun } from "./audit-run-validation.mjs"; +import { validateAuditRun, assertValidatedRunStable } from "./audit-run-validation.mjs"; export { canonicalJson } from "./canonical-json.mjs"; export { inspectRealComponents, prepareSafeOutputDirectory, assertNewOutputPath, writeNewJson, writeNewText, writeNewBytes, sha256Bytes, sha256File, readStableFile, assertStableFile, resolveInside } from "./safe-file-io.mjs"; export { loadAuditResources, defaultSkillRoot } from "./audit-resources.mjs"; export { validateArtifact, validateArtifactCandidate, validateHumanReviewCandidate } from "./audit-artifact-validation.mjs"; -export { validateAuditRun } from "./audit-run-validation.mjs"; +export { validateAuditRun, assertValidatedRunStable, validatedRunSnapshots } from "./audit-run-validation.mjs"; export { mergeArtifacts } from "./audit-run-merge.mjs"; function normalizePermission(value, aliases, name) { @@ -257,7 +257,6 @@ export function registerArtifact(run, artifact, options = {}) { const nextValidation = validateAuditRun(next, { skillRoot, runFile: options.runFile }); if (!nextValidation.valid) throw new Error(`Registered run is invalid:\n- ${nextValidation.errors.join("\n- ")}`); assertStableFile(snapshot, "artifact file"); - for (const { snapshot: registeredSnapshot } of nextValidation.envelopesById.values()) assertStableFile(registeredSnapshot, "registered artifact"); - for (const evidenceSnapshot of nextValidation.evidenceSnapshots.values()) assertStableFile(evidenceSnapshot, "raw evidence"); + assertValidatedRunStable(nextValidation); return next; } diff --git a/shared/skill/scripts/lib/finding-lifecycle.mjs b/shared/skill/scripts/lib/finding-lifecycle.mjs index 09ac048c..f706788b 100644 --- a/shared/skill/scripts/lib/finding-lifecycle.mjs +++ b/shared/skill/scripts/lib/finding-lifecycle.mjs @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { isDeepStrictEqual } from "node:util"; -import { assertNewOutputPath, assertStableFile, readStableFile, resolveInside, validateAuditRun, writeNewJson } from "./audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, resolveInside, validateAuditRun, validatedRunSnapshots, writeNewJson } from "./audit-run.mjs"; import { compareInstants, isCalendarDate } from "./date-time.mjs"; import { reviewEntries, resolveHumanReviews, consensusReviewRows, consensusRemediationItems } from "./human-review-consensus.mjs"; import { validateJsonSchema } from "./json-schema.mjs"; @@ -98,8 +98,7 @@ function validateClosure(record, run, validation, runFile) { path.resolve(successorValidation.artifactRoot, state.closure.verification_file), { existing: true }); const evidence = readStableFile(evidencePath, { label: "closure verification evidence" }); if (evidence.sha256 !== state.closure.verification_sha256) throw new Error("Closure verification evidence hash mismatch."); - return [retestSnapshot, evidence, ...[...successorValidation.envelopesById.values()].map((item) => item.snapshot), - ...successorValidation.evidenceSnapshots.values()]; + return [retestSnapshot, evidence, ...validatedRunSnapshots(successorValidation)]; } function validateRecord(record, previous, source, run, validation, runFile) { @@ -252,8 +251,7 @@ export function writeLifecycle(output, record, { run, validation, runFile, input inputSnapshots.push(...validateRecord(record, previous, source, run, validation, runFile)); const stable = () => { for (const snapshot of inputSnapshots) assertStableFile(snapshot, "lifecycle input"); - for (const item of validation.envelopesById.values()) assertStableFile(item.snapshot, "registered artifact"); - for (const snapshot of validation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(validation); }; return writeNewJson(target, record, { beforeWrite: stable }); } diff --git a/shared/skill/scripts/lib/network-cli.mjs b/shared/skill/scripts/lib/network-cli.mjs index fca04a9c..08bf7001 100644 --- a/shared/skill/scripts/lib/network-cli.mjs +++ b/shared/skill/scripts/lib/network-cli.mjs @@ -1,5 +1,5 @@ import path from "node:path"; -import { readStableFile, validateAuditRun, assertStableFile, assertNewOutputPath, writeNewJson } from "./audit-run.mjs"; +import { readStableFile, validateAuditRun, assertStableFile, assertValidatedRunStable, assertNewOutputPath, writeNewJson } from "./audit-run.mjs"; import { assertNetworkPolicy, callerNetworkScope, networkPolicyHash } from "./network-policy.mjs"; export async function withNetworkEvidenceOutput({ run, artifactRoot, output, needsNetwork, inputs = [], otherOutputs = [] }, operation) { @@ -44,8 +44,7 @@ export function prepareNetworkCapture(options, outputs = []) { options.networkCaller = caller; return { save(log) { assertStableFile(snapshot, "network run input"); - for (const record of validation.envelopesById.values()) assertStableFile(record.snapshot, "registered artifact"); - for (const raw of validation.evidenceSnapshots.values()) assertStableFile(raw, "raw evidence"); + assertValidatedRunStable(validation); writeNewJson(output, log); } }; } diff --git a/shared/skill/scripts/merge-audit-artifacts.mjs b/shared/skill/scripts/merge-audit-artifacts.mjs index 917ca4db..fcc23fc5 100644 --- a/shared/skill/scripts/merge-audit-artifacts.mjs +++ b/shared/skill/scripts/merge-audit-artifacts.mjs @@ -4,6 +4,7 @@ import { pathToFileURL } from "node:url"; import { assertStableFile, + assertValidatedRunStable, assertNewOutputPath, loadAuditResources, mergeArtifacts, @@ -91,8 +92,7 @@ export function main(argv = process.argv.slice(2)) { assertStableFile(runSnapshot, "audit run input"); assertStableFile(assessmentSnapshot, "assessment input"); for (const snapshot of artifactSnapshots) assertStableFile(snapshot, "merge artifact"); - for (const { snapshot } of runValidation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of runValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(runValidation); writeNewJson(output, merged, { beforeWrite() { for (const snapshot of [...trustInput.snapshots, ...reviewInputs.map((input) => input.snapshot)]) assertStableFile(snapshot, "review verification input"); } }); diff --git a/shared/skill/scripts/register-audit-artifact.mjs b/shared/skill/scripts/register-audit-artifact.mjs index 77b2c35e..4d2d6d90 100644 --- a/shared/skill/scripts/register-audit-artifact.mjs +++ b/shared/skill/scripts/register-audit-artifact.mjs @@ -5,6 +5,7 @@ import { withNetworkEvidenceOutput } from "./lib/network-cli.mjs"; import { assertStableFile, + assertValidatedRunStable, assertNewOutputPath, readStableFile, registerArtifactChecked, @@ -65,8 +66,7 @@ export async function main(argv = process.argv.slice(2)) { assertStableFile(artifactSnapshot, "artifact input"); const finalValidation = validateAuditRun(next, { runFile }); if (!finalValidation.valid) throw new Error(`Registered audit run failed final validation:\n- ${finalValidation.errors.join("\n- ")}`); - for (const { snapshot } of finalValidation.envelopesById.values()) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of finalValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(finalValidation); writeNewJson(output, next); process.stdout.write(`${JSON.stringify({ status: "PASS", run_id: next.run_id, run_status: next.status, artifact_id: artifact.artifact_id, output })}\n`); } diff --git a/shared/skill/scripts/render-audit-report.mjs b/shared/skill/scripts/render-audit-report.mjs index 0225c115..41d723ed 100644 --- a/shared/skill/scripts/render-audit-report.mjs +++ b/shared/skill/scripts/render-audit-report.mjs @@ -7,7 +7,7 @@ import { reviewDetailLines } from "./lib/review-details.mjs"; import path from "node:path"; import process from "node:process"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { assertStableFile, readStableFile, validateAuditRun, writeNewText } from "./lib/audit-run.mjs"; +import { assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewText } from "./lib/audit-run.mjs"; import { validateAssessment } from "./validate-assessment.mjs"; import { reviewerAssuranceText, reviewerVerificationOptions, displayedEvidenceLevel } from "./lib/assessment-provenance.mjs"; import { loadReviewTrust } from "./lib/review-trust-input.mjs"; @@ -457,13 +457,11 @@ function main() { resources: runValidation.resources, trust: trustInput.trust })); - const artifactSnapshots = [...runValidation.envelopesById.values()].map((record) => record.snapshot).filter(Boolean); const output = writeNewText(path.resolve(options.output), report, { beforeWrite() { assertStableFile(runSnapshot, "audit run"); assertStableFile(assessmentSnapshot, "merged assessment"); - for (const snapshot of artifactSnapshots) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of runValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); + assertValidatedRunStable(runValidation); for (const snapshot of trustInput.snapshots) assertStableFile(snapshot, "external reviewer trust policy"); } }); diff --git a/shared/skill/scripts/render-report.mjs b/shared/skill/scripts/render-report.mjs index c9b9de6d..c80b872e 100755 --- a/shared/skill/scripts/render-report.mjs +++ b/shared/skill/scripts/render-report.mjs @@ -8,6 +8,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { assertNewOutputPath, assertStableFile, + assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson, @@ -323,15 +324,10 @@ function renderRunBacked(options) { rendered.report = decorate(rendered.report); if (rendered.appendix) rendered.appendix = decorate(rendered.appendix); } - const artifactSnapshots = [...runValidation.envelopesById.values()] - .map((record) => record.snapshot) - .filter(Boolean); const assertInputsStable = () => { assertStableFile(runSnapshot, "audit run"); assertStableFile(assessmentSnapshot, "run-backed assessment"); - for (const snapshot of artifactSnapshots) assertStableFile(snapshot, "registered artifact"); - for (const snapshot of runValidation.evidenceSnapshots.values()) assertStableFile(snapshot, "raw evidence"); - for (const snapshot of runValidation.historicalResourceSnapshots ?? []) assertStableFile(snapshot, "historical resource"); + assertValidatedRunStable(runValidation); for (const snapshot of lifecycleSnapshots) assertStableFile(snapshot, "lifecycle input"); for (const snapshot of trustInput.snapshots) assertStableFile(snapshot, "external reviewer trust policy"); }; diff --git a/shared/skill/scripts/show-audit-status.mjs b/shared/skill/scripts/show-audit-status.mjs index f7d5bd85..3698c041 100644 --- a/shared/skill/scripts/show-audit-status.mjs +++ b/shared/skill/scripts/show-audit-status.mjs @@ -11,7 +11,7 @@ import { interactionScopeSummary } from "./lib/interaction-policy.mjs"; import { findingRelations, findingRetestSummary, remediationPlanItems } from "./lib/run-findings.mjs"; import { lifecycleSummary, loadLifecycle } from "./lib/finding-lifecycle.mjs"; import { reviewEntries, resolveHumanReviews, consensusReviewRows, consensusRemediationItems } from "./lib/human-review-consensus.mjs"; -import { assertStableFile, defaultSkillRoot, mergeArtifacts, readStableFile, validateAuditRun } from "./lib/audit-run.mjs"; +import { assertStableFile, assertValidatedRunStable, defaultSkillRoot, mergeArtifacts, readStableFile, validateAuditRun } from "./lib/audit-run.mjs"; function parse(snapshot) { return JSON.parse(snapshot.bytes.toString("utf8").replace(/^\uFEFF/u, "")); @@ -52,8 +52,7 @@ function discoverSuccessors(run, runFile, artifactRoot, skillRoot) { continue; } assertStableFile(snapshot, "sibling run candidate"); - for (const record of validation.envelopesById.values()) assertStableFile(record.snapshot, "sibling registered artifact"); - for (const evidence of validation.evidenceSnapshots.values()) assertStableFile(evidence, "sibling raw evidence"); + assertValidatedRunStable(validation, { artifact: "sibling registered artifact", evidence: "sibling raw evidence" }); if (retest || (validation.artifactRoot === artifactRoot && extendsRun(candidate, run))) { candidates.push({ file: entry.name, run_id: candidate.run_id, state: candidate.status, kind: retest ? "retest" : "registration" }); } else if (sameRun && !isDeepStrictEqual(candidate, run) && !extendsRun(run, candidate)) { @@ -161,8 +160,7 @@ export function auditStatus(runFile, { skillRoot = defaultSkillRoot, retestOf, l || !isDeepStrictEqual(run.scope, prior.scope) || !isDeepStrictEqual(run.inspection_request, prior.inspection_request)) throw new Error("Retest comparison requires the linked predecessor and the same target/profile/scope at a new version."); priorEnvelopes = [...priorValidation.envelopesById.values()].map((record) => record.envelope); comparison = true; assertStableFile(priorSnapshot, "retest predecessor"); - for (const record of priorValidation.envelopesById.values()) assertStableFile(record.snapshot, "predecessor artifact"); - for (const evidence of priorValidation.evidenceSnapshots?.values() ?? []) assertStableFile(evidence, "predecessor evidence"); + assertValidatedRunStable(priorValidation, { artifact: "predecessor artifact", evidence: "predecessor evidence", historical: "predecessor historical resource" }); } const resolve = (records) => resolveHumanReviews(reviewEntries(records.filter((item) => item.artifact_type === "declared-human-review") .map((item) => ({ payload: item.payload, artifact_id: item.artifact_id })))); @@ -177,8 +175,7 @@ export function auditStatus(runFile, { skillRoot = defaultSkillRoot, retestOf, l })); } assertStableFile(snapshot, "audit run"); - for (const record of validation.envelopesById?.values() ?? []) assertStableFile(record.snapshot, "registered artifact"); - for (const evidence of validation.evidenceSnapshots?.values() ?? []) assertStableFile(evidence, "raw evidence"); + assertValidatedRunStable(validation); return result; } diff --git a/shared/skill/scripts/validate-assessment.mjs b/shared/skill/scripts/validate-assessment.mjs index 43e2a915..c10ab984 100644 --- a/shared/skill/scripts/validate-assessment.mjs +++ b/shared/skill/scripts/validate-assessment.mjs @@ -533,7 +533,7 @@ if (isMain) { } if (!positional.length || positional.length > 5) throw new Error("Usage: node validate-assessment.mjs [registry.json] [schema.json] [catalog.json] [methods.json] [--run run.json] [--trust-policy policy.json --trust-policy-sha256 hash]"); const { loadReviewTrust, readReviewJson } = await import("./lib/review-trust-input.mjs"); - const { assertStableFile, validateAuditRun } = await import("./lib/audit-run.mjs"); + const { assertStableFile, validateAuditRun, validatedRunSnapshots } = await import("./lib/audit-run.mjs"); const { reviewerVerificationOptions } = await import("./lib/assessment-provenance.mjs"); const input = readReviewJson(positional[0], "assessment"), trustInput = loadReviewTrust(options); const snapshots = [input.snapshot, ...trustInput.snapshots]; @@ -543,7 +543,7 @@ if (isMain) { const runValidation = validateAuditRun(runInput.value, { runFile: runInput.snapshot.path }); if (!runValidation.valid) throw new Error(`Audit run validation failed: ${runValidation.errors.join("; ")}`); reviewOptions = reviewerVerificationOptions({ run: runInput.value, envelopesById: runValidation.envelopesById, trust: trustInput.trust }); - snapshots.push(runInput.snapshot, ...[...runValidation.envelopesById.values()].map((item) => item.snapshot), ...runValidation.evidenceSnapshots.values()); + snapshots.push(runInput.snapshot, ...validatedRunSnapshots(runValidation)); } const references = ["standards-registry.json", "assessment-record.schema.json", "criteria-catalog.json", "web-audit-methods.json"] .map((name, index) => readJson(positional[index + 1] ?? fileURLToPath(new URL(`../references/${name}`, import.meta.url)))); diff --git a/shared/skill/scripts/validate-audit-run.mjs b/shared/skill/scripts/validate-audit-run.mjs index dfc58907..dbd5fc45 100644 --- a/shared/skill/scripts/validate-audit-run.mjs +++ b/shared/skill/scripts/validate-audit-run.mjs @@ -2,7 +2,7 @@ import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; -import { assertNewOutputPath, assertStableFile, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; +import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; function parseArgs(argv) { const options = {}; @@ -32,9 +32,7 @@ export function main(argv = process.argv.slice(2)) { const snapshot = readStableFile(input, { label: "audit run input" }); const result = validateAuditRun(parseSnapshot(snapshot), { runFile: input, historicalResourceRoot: options.historicalResources }); assertStableFile(snapshot, "audit run input"); - for (const historical of result.historicalResourceSnapshots ?? []) assertStableFile(historical, "historical resource"); - for (const { snapshot: artifact } of result.envelopesById?.values() ?? []) assertStableFile(artifact, "registered artifact"); - for (const evidence of result.evidenceSnapshots?.values() ?? []) assertStableFile(evidence, "raw evidence"); + assertValidatedRunStable(result); writeNewJson(output, { valid: result.valid, errors: result.errors }); process.stdout.write(`${JSON.stringify({ status: result.valid ? "PASS" : "FAIL", input, output, valid: result.valid })}\n`); if (!result.valid) { diff --git a/tests/validated-run-stability.test.mjs b/tests/validated-run-stability.test.mjs new file mode 100644 index 00000000..8e40c6f4 --- /dev/null +++ b/tests/validated-run-stability.test.mjs @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { assertValidatedRunStable, validatedRunSnapshots } from "../codex/skills/information-accessibility-practice/scripts/lib/audit-run.mjs"; +import { readStableFile } from "../codex/skills/information-accessibility-practice/scripts/lib/safe-file-io.mjs"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +// A stand-in for what validateAuditRun returns: one registered artifact, one saved evidence file, one historical resource. +function fixture(t) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "a11y-validated-run-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const files = Object.fromEntries(["artifact", "evidence", "historical"].map((name) => { + const file = path.join(directory, `${name}.json`); + fs.writeFileSync(file, JSON.stringify({ name })); + return [name, file]; + })); + const validation = { + valid: true, + envelopesById: new Map([["ARTIFACT-1", { envelope: {}, snapshot: readStableFile(files.artifact) }]]), + evidenceSnapshots: new Map([["evidence.json", readStableFile(files.evidence)]]), + historicalResourceSnapshots: [readStableFile(files.historical)] + }; + return { files, validation }; +} + +test("a validated run passes while nothing it read has changed", (t) => { + const { validation } = fixture(t); + assert.doesNotThrow(() => assertValidatedRunStable(validation)); + assert.equal(validatedRunSnapshots(validation).length, 3); +}); + +for (const [name, label] of [["artifact", "registered artifact"], ["evidence", "raw evidence"], ["historical", "historical resource"]]) { + test(`a change to the ${name} input after validation is refused with its label`, (t) => { + const { files, validation } = fixture(t); + fs.writeFileSync(files[name], JSON.stringify({ name, edited: true })); + assert.throws(() => assertValidatedRunStable(validation), new RegExp(`${label} changed before commit`, "u")); + }); +} + +test("callers can rename the inputs, and a validation without snapshots is tolerated", (t) => { + const { files, validation } = fixture(t); + fs.writeFileSync(files.evidence, "{}"); + assert.throws(() => assertValidatedRunStable(validation, { evidence: "saved evidence" }), /saved evidence changed before commit/u); + assert.doesNotThrow(() => assertValidatedRunStable({ valid: false, errors: ["rejected"] })); + assert.deepEqual(validatedRunSnapshots({ valid: false, errors: ["rejected"] }), []); +}); + +function sourceFiles(directory) { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(directory, entry.name); + if (entry.isDirectory()) return sourceFiles(full); + return entry.name.endsWith(".mjs") ? [full] : []; + }); +} + +test("no command re-implements the post-validation re-check by hand", () => { + const helper = path.join(root, "shared/skill/scripts/lib/audit-run-validation.mjs"); + const offenders = sourceFiles(path.join(root, "shared/skill/scripts")).filter((file) => file !== helper).flatMap((file) => + fs.readFileSync(file, "utf8").split(/\r?\n/u).flatMap((line, index) => + /assertStableFile\(/u.test(line) && /\b(?:envelopesById|evidenceSnapshots|historicalResourceSnapshots)\b/u.test(line) + ? [`${path.relative(root, file).split(path.sep).join("/")}:${index + 1}`] : [])); + assert.deepEqual(offenders, [], "use assertValidatedRunStable or validatedRunSnapshots from lib/audit-run.mjs"); +});