diff --git a/CHANGELOG.md b/CHANGELOG.md index b16c5b7..c11774d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ This project uses a Keep a Changelog-style structure. Until a tagged release is ### Changed +- `compare-evidence` now accepts `--output` only inside the successor run's private artifact root, as `compare-runs` and `compare-targets` already did, and checks it after both runs validate so a rejected path no longer creates parent directories. Scripts that wrote the comparison elsewhere must point `--output` at the `--after` run's artifact root; the saved-evidence example and the command usage line show the new form. + - Node.js 22 or later is now required (package `engines`, `doctor`, the Web capability contract and the documentation). Node.js 20 reached end of life on 2026-04-30 and is no longer supported or tested. CI runs Node.js 22 and 24 on Ubuntu, Windows and macOS; workflow actions move to versions that run on Node.js 24 and are pinned by commit, and the browser E2E workflows pin Ubuntu 24.04. Contributors install test dependencies with `npm install` through a root `package.json`, which also stops an unrelated parent-directory `package.json` from capturing the install. - HTTP target observation always runs through the logged, size-bounded network session under the saved run policy. The second retrieval path, reachable only by direct library callers, did not reject content encodings, cap total bytes or keep a request log; it was removed. No CLI command used it. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 1031ef8..887f4be 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -512,7 +512,7 @@ Notes: Compare validated saved evidence before and after a change, for private review. Usage: - accessibility-audit compare-evidence --before --after --output + accessibility-audit compare-evidence --before --after --output Notes: - Both runs and their saved files are revalidated. Results describe byte changes, not accessibility outcomes. @@ -524,7 +524,7 @@ Notes: 変更前後の検証済み保存証拠を、非公開レビュー用に比較します。 使用方法: - accessibility-audit compare-evidence --before --after --output + accessibility-audit compare-evidence --before --after --output 注意: - Both runs and their saved files are revalidated. Results describe byte changes, not accessibility outcomes. diff --git a/shared/skill/references/saved-evidence.md b/shared/skill/references/saved-evidence.md index 3b68d29..23ef063 100644 --- a/shared/skill/references/saved-evidence.md +++ b/shared/skill/references/saved-evidence.md @@ -84,10 +84,12 @@ in addition to registered artifact snapshots. ## Private before/after comparison ```text -node /scripts/accessibility-audit.mjs compare-evidence --before before/run.json --after after/run.json --output comparison.private.json +node /scripts/accessibility-audit.mjs compare-evidence --before before/run.json --after after/run.json --output after/artifacts/comparison.private.json ``` -Both runs and their referenced files must validate. Results group captures by +The output must be a new file inside the after-run's private artifact root +(`after/artifacts` in this example). Both runs and their referenced files must +validate. Results group captures by screening requirement, exact target reference and evidence type. Status is `added`, `removed`, `bytes_changed` or `bytes_unchanged`. `context_changed` separately identifies changes to the declared version, target context or environment, even diff --git a/shared/skill/scripts/compare-audit-evidence.mjs b/shared/skill/scripts/compare-audit-evidence.mjs index 2d2bd50..e016fdb 100644 --- a/shared/skill/scripts/compare-audit-evidence.mjs +++ b/shared/skill/scripts/compare-audit-evidence.mjs @@ -3,6 +3,14 @@ import { pathToFileURL } from "node:url"; import { assertNewOutputPath, assertStableFile, assertValidatedRunStable, readStableFile, validateAuditRun, writeNewJson } from "./lib/audit-run.mjs"; import { compareEvidenceReferences } from "./lib/run-evidence.mjs"; +function insidePrivateRoot(root, file) { + const absolute = path.resolve(file), relative = path.relative(root, absolute); + if (!relative || path.isAbsolute(relative) || relative === ".." || relative.startsWith(`..${path.sep}`)) { + throw new Error("Comparison outputs must stay inside the successor private artifact root."); + } + return assertNewOutputPath(absolute); +} + export function main(argv = process.argv.slice(2)) { const options = {}; for (let i = 0; i < argv.length; i += 2) { @@ -12,7 +20,6 @@ export function main(argv = process.argv.slice(2)) { options[argv[i]] = argv[i + 1]; } for (const flag of ["--before", "--after", "--output"]) if (!options[flag]) throw new Error(`${flag} is required`); - assertNewOutputPath(options["--output"]); const inputs = [options["--before"], options["--after"]].map((file) => { const snapshot = readStableFile(file); const run = JSON.parse(snapshot.bytes.toString("utf8").replace(/^\uFEFF/u, "")); @@ -23,6 +30,7 @@ export function main(argv = process.argv.slice(2)) { return { run, snapshot, validation, rows: screenings.flatMap((artifact) => artifact.payload.observations) }; }); const [before, after] = inputs; + insidePrivateRoot(after.validation.artifactRoot, options["--output"]); const result = { schema_version: "1.0.0", publication: "private_by_default", before_run_id: before.run.run_id, after_run_id: after.run.run_id, limitation: "Byte differences are not accessibility judgements or proof that a live target still matches its capture.", comparisons: compareEvidenceReferences(before.rows, after.rows) }; diff --git a/shared/skill/scripts/lib/cli-command-registry.mjs b/shared/skill/scripts/lib/cli-command-registry.mjs index f1d194b..eb064b4 100644 --- a/shared/skill/scripts/lib/cli-command-registry.mjs +++ b/shared/skill/scripts/lib/cli-command-registry.mjs @@ -119,7 +119,7 @@ const definitions = [ name: "compare-evidence", script: "compare-audit-evidence.mjs", summary: "Compare validated saved evidence before and after a change, for private review.", - usage: ["accessibility-audit compare-evidence --before --after --output "], + usage: ["accessibility-audit compare-evidence --before --after --output "], notes: ["Both runs and their saved files are revalidated. Results describe byte changes, not accessibility outcomes."] }, { diff --git a/tests/run-evidence.test.mjs b/tests/run-evidence.test.mjs index d37ec94..b63af5e 100644 --- a/tests/run-evidence.test.mjs +++ b/tests/run-evidence.test.mjs @@ -172,15 +172,19 @@ test("bind-evidence creates a new artifact from captured bytes without changing assert.doesNotThrow(() => registerArtifact(f.run, bound, { runFile: f.runFile, artifactFile: output })); }); -test("comparison records before/after bytes, dates and versions without interpreting outcomes", (t) => { +test("comparison writes inside the after-run private artifact root without interpreting outcomes", (t) => { const before = fixture(t); const after = fixture(t, { suffix: "0002", version: "release-2", bytes: Buffer.from("
Changed content
") }); const beforeFile = path.join(before.temp, "registered.json"); const afterFile = path.join(after.temp, "registered.json"); writeNewJson(beforeFile, before.register()); writeNewJson(afterFile, after.register()); - const output = path.join(after.temp, "comparison.json"); - pass(cli(["compare-evidence", "--before", beforeFile, "--after", afterFile, "--output", output])); + const output = path.join(after.artifactRoot, "comparison.json"); + const args = ["compare-evidence", "--before", beforeFile, "--after", afterFile, "--output", output]; + pass(cli(args)); + const saved = fs.readFileSync(output); + assert.equal(cli(args).status, 1, "existing output is never overwritten"); + assert.deepEqual(fs.readFileSync(output), saved); const result = json(output); assert.equal(result.publication, "private_by_default"); assert.equal(result.comparisons[0].status, "bytes_changed"); @@ -200,3 +204,19 @@ test("comparison records before/after bytes, dates and versions without interpre const evidence = collectScreeningEvidence(before.run, [before.artifact], () => ({ bytes: before.bytes, sha256: hash(before.bytes) })); assert.deepEqual(evidence.errors, []); }); + +test("comparison rejects outputs outside the successor private artifact root without writing", (t) => { + const before = fixture(t); + const after = fixture(t, { suffix: "0002" }); + const beforeFile = path.join(before.temp, "registered.json"); + const afterFile = path.join(after.temp, "registered.json"); + writeNewJson(beforeFile, before.register()); + writeNewJson(afterFile, after.register()); + for (const output of [path.join(after.temp, "comparison.json"), path.join(before.artifactRoot, "comparison.json"), path.join(`${after.artifactRoot}-outside`, "comparison.json")]) { + const result = cli(["compare-evidence", "--before", beforeFile, "--after", afterFile, "--output", output]); + assert.equal(result.status, 1); + assert.match(result.stderr, /Comparison outputs must stay inside the successor private artifact root\./); + assert.equal(fs.existsSync(output), false); + } + assert.equal(fs.existsSync(`${after.artifactRoot}-outside`), false); +});