From d5e3d9cec63639cb74bd18066789e0931a9c7f57 Mon Sep 17 00:00:00 2001 From: nasarajoe Date: Mon, 19 Oct 2026 12:45:59 +0100 Subject: [PATCH] feat/utix-253-260-262-263 --- .../registryManifestValidation.test.ts | 2 +- core/operations/__tests__/recovery.test.ts | 38 ++++++ core/operations/index.ts | 2 + core/operations/recovery.ts | 35 +++++ core/operations/types.ts | 1 + .../__tests__/maintainerSimulation.test.ts | 19 +++ core/policy/maintainerSimulation.ts | 24 +++- core/policy/simulation.ts | 4 +- core/sorting/__tests__/mixedStatus.test.ts | 45 +++++++ core/sorting/index.ts | 9 ++ core/sorting/mixedStatus.ts | 76 +++++++++++ .../__tests__/recoveryFixtures.test.ts | 60 +++++++++ core/testing/index.ts | 10 ++ core/testing/recoveryFixtures.ts | 126 ++++++++++++++++++ docs/DETERMINISTIC_SORTING.md | 21 +++ docs/OPERATION_RECOVERY.md | 6 + docs/POLICY_SIMULATION.md | 9 ++ docs/RECOVERY_FIXTURES.md | 22 +++ 18 files changed, 506 insertions(+), 3 deletions(-) create mode 100644 core/sorting/__tests__/mixedStatus.test.ts create mode 100644 core/sorting/index.ts create mode 100644 core/sorting/mixedStatus.ts create mode 100644 core/testing/__tests__/recoveryFixtures.test.ts create mode 100644 core/testing/recoveryFixtures.ts create mode 100644 docs/DETERMINISTIC_SORTING.md create mode 100644 docs/RECOVERY_FIXTURES.md diff --git a/core/__tests__/registryManifestValidation.test.ts b/core/__tests__/registryManifestValidation.test.ts index c4ae49d..e474494 100644 --- a/core/__tests__/registryManifestValidation.test.ts +++ b/core/__tests__/registryManifestValidation.test.ts @@ -21,7 +21,7 @@ import { * * These fixtures are `.txt` rather than `.ts` on purpose: a manifest declaring * an invalid status or network is exactly what the type system rejects, and - * `tsconfig.json` includes `**/*.ts`, so typed fixtures could not express the + * `tsconfig.json` includes every TypeScript file, so typed fixtures could not express the * invalid shapes at all. */ const FIXTURE_DIR = path.resolve(process.cwd(), "scripts/__tests__/fixtures/registry-manifests"); diff --git a/core/operations/__tests__/recovery.test.ts b/core/operations/__tests__/recovery.test.ts index 2fcdc97..9b7b4cf 100644 --- a/core/operations/__tests__/recovery.test.ts +++ b/core/operations/__tests__/recovery.test.ts @@ -2,12 +2,14 @@ import { describe, expect, it } from "vitest"; import type { Result } from "@/core/result/result"; import { abandonOperation, + abandonOperationAuthorized, createOperationStorage, decideStep, defaultIdempotencyKey, nextActionFor, operationMachine, resolveInterruptedStep, + resolveInterruptedStepAuthorized, runOperation, type OperationDefinition, type OperationErrorCode, @@ -382,6 +384,42 @@ describe("runOperation", () => { }); }); +describe("authorized recovery decisions", () => { + it("fails closed without changing the checkpoint", async () => { + const storage = createOperationStorage(); + const definition: OperationDefinition = { + kind: "ticket_flow", + steps: [{ id: "submit", run: () => ({ status: "interrupted", errorCode: "timeout", committed: true }) }] + }; + await runOperation(definition, options(storage)); + const before = storage.read("op-1"); + + expect(resolveInterruptedStepAuthorized( + { role: "viewer", userId: "eve" }, "op-1", "submit", "applied", { storage } + )).toEqual({ ok: false, code: "permission_denied" }); + expect(abandonOperationAuthorized({ role: "service" }, "op-1", { storage })).toEqual({ + ok: false, + code: "permission_denied" + }); + expect(storage.read("op-1")).toEqual(before); + }); + + it("allows a maintainer to resolve and resume", async () => { + const storage = createOperationStorage(); + const definition: OperationDefinition = { + kind: "ticket_flow", + steps: [{ id: "submit", run: () => ({ status: "interrupted", errorCode: "timeout", committed: true }) }] + }; + await runOperation(definition, options(storage)); + + const resolved = resolveInterruptedStepAuthorized( + { role: "maintainer", userId: "ada" }, "op-1", "submit", "applied", { storage } + ); + expect(resolved.ok && resolved.value.steps[0].status).toBe("succeeded"); + expect(progress(await runOperation(definition, options(storage, 1))).state).toBe("completed"); + }); +}); + describe("resolveInterruptedStep", () => { it('marks a confirmed effect as applied and lets the flow continue', async () => { const log = recorder(); diff --git a/core/operations/index.ts b/core/operations/index.ts index c46302c..7529c6c 100644 --- a/core/operations/index.ts +++ b/core/operations/index.ts @@ -9,11 +9,13 @@ export { export { abandonOperation, + abandonOperationAuthorized, decideStep, defaultIdempotencyKey, nextActionFor, operationMachine, resolveInterruptedStep, + resolveInterruptedStepAuthorized, runOperation, type ResolveOperationOptions, type StepDecision, diff --git a/core/operations/recovery.ts b/core/operations/recovery.ts index 9b2960a..49f61d2 100644 --- a/core/operations/recovery.ts +++ b/core/operations/recovery.ts @@ -24,6 +24,8 @@ import { type TransitionContext } from "@/core/lifecycle/lifecycle"; import { newCorrelationId } from "@/core/telemetry/telemetry"; +import type { AuthContext } from "@/core/auth/middleware"; +import { hasPermission } from "@/core/auth/types"; import { err, isErr, ok, type Result } from "@/core/result/result"; import { OPERATION_STATES, @@ -504,6 +506,26 @@ export function resolveInterruptedStep( return ok(next); } +/** + * Server-side boundary for the destructive recovery decision. The original + * resolver remains available for trusted/internal callers; request handlers + * should use this function so a viewer cannot assert that an external effect + * did or did not land. + */ +export function resolveInterruptedStepAuthorized( + auth: AuthContext, + operationId: string, + stepId: string, + resolution: StepResolution, + options: ResolveOperationOptions +): Result { + if (!hasPermission(auth.role, "admin:recovery")) return err("permission_denied"); + return resolveInterruptedStep(operationId, stepId, resolution, { + ...options, + actor: options.actor ?? `maintainer:${auth.userId ?? "unknown"}` + }); +} + /** Gives up on an operation on purpose. The record is kept for the audit trail. */ export function abandonOperation( operationId: string, @@ -524,3 +546,16 @@ export function abandonOperation( if (!persist(options.storage, advanced.value)) return err("storage_error"); return ok(advanced.value); } + +/** Authorization boundary for abandoning a recoverable operation. */ +export function abandonOperationAuthorized( + auth: AuthContext, + operationId: string, + options: ResolveOperationOptions +): Result { + if (!hasPermission(auth.role, "admin:recovery")) return err("permission_denied"); + return abandonOperation(operationId, { + ...options, + actor: options.actor ?? `maintainer:${auth.userId ?? "unknown"}` + }); +} diff --git a/core/operations/types.ts b/core/operations/types.ts index 39f6d39..14f5420 100644 --- a/core/operations/types.ts +++ b/core/operations/types.ts @@ -198,4 +198,5 @@ export type OperationErrorCode = | "unknown_operation" | "unknown_step" | "terminal_operation" + | "permission_denied" | "storage_error"; diff --git a/core/policy/__tests__/maintainerSimulation.test.ts b/core/policy/__tests__/maintainerSimulation.test.ts index 567c7df..163d2eb 100644 --- a/core/policy/__tests__/maintainerSimulation.test.ts +++ b/core/policy/__tests__/maintainerSimulation.test.ts @@ -67,6 +67,11 @@ describe("runPolicySimulation", () => { expect(JSON.stringify({ POLICY, records })).toBe(before); }); + it("returns an identical preview when a dry run is retried", () => { + const request = { policy: POLICY, records: [tx("tx-1", 5000)], now: 1_700_000_000_000 }; + expect(run(request)).toEqual(run(request)); + }); + it("rejects an invalid policy with every problem listed", () => { const bad = { ...POLICY, id: "", version: 0, parameters: { maxBaseFeeStroops: -1 } }; const result = runPolicySimulation(MAINTAINER, { policy: bad, records: RECORDS }); @@ -81,6 +86,20 @@ describe("runPolicySimulation", () => { expect(!result.ok && result.detail).toEqual(['duplicate record id "tx-1"', 'record "x" has an unknown type']); }); + it("rejects malformed preview bounds without evaluating", () => { + const duplicateSelection = runPolicySimulation(MAINTAINER, { + policy: POLICY, + records: RECORDS, + recordIds: ["tx-1", "tx-1"], + maxExamples: -1 + }); + expect(duplicateSelection).toMatchObject({ ok: false, code: "invalid_records" }); + expect(!duplicateSelection.ok && duplicateSelection.detail).toEqual([ + "recordIds must not contain duplicates", + "maxExamples must be a non-negative integer" + ]); + }); + it("denies roles without config access", () => { for (const role of ["viewer", "service"] as const) { expect(runPolicySimulation({ role }, { policy: POLICY, records: RECORDS })).toEqual({ diff --git a/core/policy/maintainerSimulation.ts b/core/policy/maintainerSimulation.ts index dbdb1a0..96d2eb1 100644 --- a/core/policy/maintainerSimulation.ts +++ b/core/policy/maintainerSimulation.ts @@ -134,6 +134,26 @@ function recordIssues(records: unknown): string[] { return issues; } +function requestIssues(request: MaintainerSimulationRequest): string[] { + const issues: string[] = []; + if (request.recordIds !== undefined) { + if (!Array.isArray(request.recordIds) || !request.recordIds.every(isNonEmptyString)) { + issues.push("recordIds must be an array of non-empty strings"); + } else if (new Set(request.recordIds).size !== request.recordIds.length) { + issues.push("recordIds must not contain duplicates"); + } + } + if (request.maxExamples !== undefined && + (!Number.isInteger(request.maxExamples) || request.maxExamples < 0)) { + issues.push("maxExamples must be a non-negative integer"); + } + const now = request.now instanceof Date ? request.now.getTime() : request.now; + if (now !== undefined && (!Number.isFinite(now) || !Number.isSafeInteger(now))) { + issues.push("now must be a valid Date or integer timestamp"); + } + return issues; +} + function buildExamples(report: PolicySimulationReport): AffectedRecordExample[] { const byRecord = new Map(); for (const effect of report.sideEffects) { @@ -151,7 +171,7 @@ function buildExamples(report: PolicySimulationReport): AffectedRecordExample[] }); } return [...byRecord.values()].sort( - (a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity] || (a.id < b.id ? -1 : 1) + (a, b) => SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity] || a.id.localeCompare(b.id) ); } @@ -165,6 +185,8 @@ export function runPolicySimulation( if (policyProblems.length > 0) return err("invalid_policy", policyProblems); const recordProblems = recordIssues(request.records); if (recordProblems.length > 0) return err("invalid_records", recordProblems); + const selectionProblems = requestIssues(request); + if (selectionProblems.length > 0) return err("invalid_records", selectionProblems); const wanted = request.recordIds ? new Set(request.recordIds) : undefined; const selected = wanted ? request.records.filter((record) => wanted.has(record.id)) : request.records; diff --git a/core/policy/simulation.ts b/core/policy/simulation.ts index c9a8085..db337c8 100644 --- a/core/policy/simulation.ts +++ b/core/policy/simulation.ts @@ -33,7 +33,9 @@ export class PolicySimulator { ? new Date(options.now).toISOString() : new Date().toISOString(); - const simulationId = `sim-${policy.id}-v${policy.version}-${Date.now().toString(16)}`; + // Derive the id from the injected evaluation time. Retrying the same dry + // run against the same snapshot can therefore be compared byte-for-byte. + const simulationId = `sim-${policy.id}-v${policy.version}-${new Date(timestamp).getTime().toString(16)}`; const sideEffects: SimulationSideEffect[] = []; const affectedRecordIds = new Set(); diff --git a/core/sorting/__tests__/mixedStatus.test.ts b/core/sorting/__tests__/mixedStatus.test.ts new file mode 100644 index 0000000..745cb28 --- /dev/null +++ b/core/sorting/__tests__/mixedStatus.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { sortMixedStatusEvents, sortMixedStatusRecords } from "@/core/sorting"; + +describe("mixed-status deterministic sorting", () => { + it("orders records by actionable status and deterministic tie-breakers", () => { + const input = [ + { id: "done", status: "completed", updatedAt: "2026-01-03T00:00:00Z" }, + { id: "failure-b", status: "failed", updatedAt: "2026-01-02T00:00:00Z", sequence: 1 }, + { id: "pending", status: "pending", updatedAt: "2026-01-04T00:00:00Z" }, + { id: "failure-a", status: "failed", updatedAt: "2026-01-02T00:00:00Z", sequence: 2 } + ]; + expect(sortMixedStatusRecords(input).map(({ id }) => id)).toEqual([ + "failure-a", "failure-b", "pending", "done" + ]); + expect(input.map(({ id }) => id)).toEqual(["done", "failure-b", "pending", "failure-a"]); + }); + + it("keeps events reverse chronological across statuses", () => { + const events = [ + { id: "a", status: "completed", at: 10, sequence: 1 }, + { id: "c", status: "running", at: 20, sequence: 1 }, + { id: "b", status: "failed", at: 10, sequence: 1 } + ]; + expect(sortMixedStatusEvents(events).map(({ id }) => id)).toEqual(["c", "b", "a"]); + }); + + it("sorts unknown statuses and invalid dates without relying on input order", () => { + const records = [ + { id: "z", status: "future", updatedAt: "bad" }, + { id: "a", status: "future", updatedAt: null }, + { id: "m", status: "experimental", updatedAt: undefined } + ]; + expect(sortMixedStatusRecords(records).map(({ id }) => id)).toEqual(["m", "a", "z"]); + expect(sortMixedStatusRecords([...records].reverse()).map(({ id }) => id)).toEqual(["m", "a", "z"]); + }); + + it("only returns the caller-authorized input subset", () => { + const all = [ + { id: "visible", status: "pending", updatedAt: 1, owner: "ada" }, + { id: "secret", status: "failed", updatedAt: 2, owner: "eve" } + ]; + const authorized = all.filter((record) => record.owner === "ada"); + expect(sortMixedStatusRecords(authorized).map(({ id }) => id)).toEqual(["visible"]); + }); +}); diff --git a/core/sorting/index.ts b/core/sorting/index.ts new file mode 100644 index 0000000..f3e6180 --- /dev/null +++ b/core/sorting/index.ts @@ -0,0 +1,9 @@ +export { + STATUS_PRIORITY, + compareMixedStatusEvents, + compareMixedStatusRecords, + sortMixedStatusEvents, + sortMixedStatusRecords, + type MixedStatusEvent, + type MixedStatusRecord +} from "./mixedStatus"; diff --git a/core/sorting/mixedStatus.ts b/core/sorting/mixedStatus.ts new file mode 100644 index 0000000..faec445 --- /dev/null +++ b/core/sorting/mixedStatus.ts @@ -0,0 +1,76 @@ +/** A stable, total order for operational records and their event streams. */ + +export interface MixedStatusRecord { + readonly id: string; + readonly status: string; + readonly updatedAt?: string | number | Date | null; + readonly sequence?: number | null; +} + +export interface MixedStatusEvent { + readonly id: string; + readonly status: string; + readonly at?: string | number | Date | null; + readonly sequence?: number | null; +} + +/** Actionable/unsafe states precede in-flight and terminal states. */ +export const STATUS_PRIORITY: Readonly> = Object.freeze({ + failed: 0, + dead_lettered: 0, + interrupted: 1, + retrying: 1, + pending: 2, + queued: 2, + running: 3, + succeeded: 4, + completed: 4, + resolved: 4, + skipped: 5, + abandoned: 6, + cancelled: 6 +}); + +function text(a: string, b: string): number { + return a === b ? 0 : a < b ? -1 : 1; +} + +function time(value: string | number | Date | null | undefined): number { + if (value === null || value === undefined) return Number.NEGATIVE_INFINITY; + const parsed = value instanceof Date ? value.getTime() : typeof value === "number" ? value : Date.parse(value); + return Number.isFinite(parsed) ? parsed : Number.NEGATIVE_INFINITY; +} + +function priority(status: string): number { + return STATUS_PRIORITY[status] ?? 7; +} + +function sequence(value: number | null | undefined): number { + return typeof value === "number" && Number.isFinite(value) ? value : Number.NEGATIVE_INFINITY; +} + +/** Status first, then newest update, highest sequence, status text and id. */ +export function compareMixedStatusRecords(a: T, b: T): number { + return priority(a.status) - priority(b.status) || + time(b.updatedAt) - time(a.updatedAt) || + sequence(b.sequence) - sequence(a.sequence) || + text(a.status, b.status) || + text(a.id, b.id); +} + +/** Newest event first, then status, sequence and id for timestamp ties. */ +export function compareMixedStatusEvents(a: T, b: T): number { + return time(b.at) - time(a.at) || + priority(a.status) - priority(b.status) || + sequence(b.sequence) - sequence(a.sequence) || + text(a.status, b.status) || + text(a.id, b.id); +} + +export function sortMixedStatusRecords(records: readonly T[]): T[] { + return [...records].sort(compareMixedStatusRecords); +} + +export function sortMixedStatusEvents(events: readonly T[]): T[] { + return [...events].sort(compareMixedStatusEvents); +} diff --git a/core/testing/__tests__/recoveryFixtures.test.ts b/core/testing/__tests__/recoveryFixtures.test.ts new file mode 100644 index 0000000..de68ec4 --- /dev/null +++ b/core/testing/__tests__/recoveryFixtures.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { + decideStep, + nextActionFor, + resolveInterruptedStepAuthorized, + runOperation, + type OperationDefinition +} from "@/core/operations"; +import { + createFaultingOperationStorage, + generateRecoveryFixtureCorpus, + generateRecoveryPathFixture +} from "@/core/testing/recoveryFixtures"; + +describe("recovery fixture generators", () => { + it("is seeded, serializable, and covers every recovery path", () => { + const first = generateRecoveryFixtureCorpus(42); + const again = generateRecoveryFixtureCorpus(42); + expect(JSON.stringify(first)).toBe(JSON.stringify(again)); + expect(first.map(({ scenario }) => scenario)).toEqual([ + "normal", "pre_effect_failure", "retry", "ambiguous_commit", "authorization_denied" + ]); + }); + + it.each(generateRecoveryFixtureCorpus(7))("matches the $scenario recovery contract", (fixture) => { + const checkpoint = fixture.record.steps[0]; + const decision = fixture.record.state === "completed" ? "complete" : decideStep(checkpoint); + expect(decision).toBe(fixture.expected.decision); + expect(nextActionFor(fixture.record)?.code ?? null).toBe(fixture.expected.action); + }); + + it("drives the real authorization denial without mutating the fixture", () => { + const fixture = generateRecoveryPathFixture({ scenario: "authorization_denied", seed: 9 }); + const storage = createFaultingOperationStorage(fixture.record, "remove"); + const before = JSON.stringify(storage.read(fixture.record.id)); + expect(resolveInterruptedStepAuthorized( + fixture.auth, fixture.record.id, "external-effect", "applied", { storage } + )).toEqual({ ok: false, code: "permission_denied" }); + expect(JSON.stringify(storage.read(fixture.record.id))).toBe(before); + }); + + it("drives a checkpoint write failure through the real runner", async () => { + const storage = createFaultingOperationStorage(undefined, "write"); + const definition: OperationDefinition = { + kind: "fixture", + steps: [{ id: "external-effect", run: () => ({ status: "succeeded" }) }] + }; + await expect(runOperation(definition, { + operationId: "faulting-write", + context: null, + storage, + now: () => 1_700_000_000_000, + newCorrelationId: () => "corr-fixture" + })).resolves.toEqual({ ok: false, code: "storage_error" }); + }); + + it("rejects unknown scenarios instead of silently generating misleading data", () => { + expect(() => generateRecoveryPathFixture({ scenario: "typo" as never })).toThrow(TypeError); + }); +}); diff --git a/core/testing/index.ts b/core/testing/index.ts index e1024cc..07204a7 100644 --- a/core/testing/index.ts +++ b/core/testing/index.ts @@ -44,6 +44,16 @@ export { type NetworkProfileContext } from "@/core/testing/networkProfiles"; export { networkProfileHandlers } from "@/core/testing/networkProfileHandlers"; +export { + RECOVERY_FIXTURE_SCENARIOS, + createFaultingOperationStorage, + generateRecoveryFixtureCorpus, + generateRecoveryPathFixture, + type RecoveryFixtureOptions, + type RecoveryFixtureScenario, + type RecoveryPathFixture, + type StorageFailurePoint +} from "@/core/testing/recoveryFixtures"; export { runAgainstNetworkProfiles, getProfileEdgeCases, diff --git a/core/testing/recoveryFixtures.ts b/core/testing/recoveryFixtures.ts new file mode 100644 index 0000000..d890520 --- /dev/null +++ b/core/testing/recoveryFixtures.ts @@ -0,0 +1,126 @@ +import type { + OperationRecord, + OperationStorage, + RecoveryActionCode, + StepDecision, + StepStatus +} from "@/core/operations"; +import type { AuthContext } from "@/core/auth/middleware"; + +export const RECOVERY_FIXTURE_SCENARIOS = [ + "normal", + "pre_effect_failure", + "retry", + "ambiguous_commit", + "authorization_denied" +] as const; + +export type RecoveryFixtureScenario = typeof RECOVERY_FIXTURE_SCENARIOS[number]; + +export interface RecoveryPathFixture { + readonly scenario: RecoveryFixtureScenario; + readonly auth: AuthContext; + readonly record: OperationRecord; + readonly expected: { + readonly decision: StepDecision | "complete"; + readonly action: RecoveryActionCode | null; + readonly mayResolve: boolean; + }; +} + +export interface RecoveryFixtureOptions { + readonly scenario: RecoveryFixtureScenario; + /** Any integer; equal seeds produce equal JSON fixtures. */ + readonly seed?: number; +} + +function seeded(seed: number): () => number { + let state = (Number.isFinite(seed) ? Math.trunc(seed) : 1) >>> 0; + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state; + }; +} + +/** Generates plain-data fixtures; no timers, randomness, storage, or network. */ +export function generateRecoveryPathFixture({ scenario, seed = 1 }: RecoveryFixtureOptions): RecoveryPathFixture { + if (!RECOVERY_FIXTURE_SCENARIOS.includes(scenario)) { + throw new TypeError(`unknown recovery fixture scenario: ${String(scenario)}`); + } + const next = seeded(seed); + const suffix = next().toString(16).padStart(8, "0"); + const at = new Date(1_700_000_000_000 + (next() % 86_400_000)).toISOString(); + const operationId = `fixture-${suffix}`; + + const stateByScenario = { + normal: "completed", + pre_effect_failure: "failed", + retry: "interrupted", + ambiguous_commit: "interrupted", + authorization_denied: "interrupted" + } as const; + const statusByScenario: Record = { + normal: "succeeded", + pre_effect_failure: "failed", + retry: "interrupted", + ambiguous_commit: "interrupted", + authorization_denied: "interrupted" + }; + const committed = scenario === "normal" || scenario === "ambiguous_commit" || scenario === "authorization_denied"; + const decision = scenario === "normal" ? "complete" : committed ? "confirm" : "run"; + + return { + scenario, + auth: scenario === "authorization_denied" + ? { role: "viewer", userId: `viewer-${suffix}` } + : { role: "maintainer", userId: `maintainer-${suffix}` }, + record: { + id: operationId, + kind: "fixture_external_effect", + state: stateByScenario[scenario], + correlationId: `corr-${suffix}`, + createdAt: at, + updatedAt: at, + lastErrorCode: scenario === "normal" ? undefined : scenario, + steps: [{ + id: "external-effect", + status: statusByScenario[scenario], + attempt: scenario === "retry" ? 2 : 1, + idempotencyKey: `operation:${operationId}:step:external-effect`, + sideEffectCommitted: committed, + updatedAt: at, + errorCode: scenario === "normal" ? undefined : scenario + }] + }, + expected: { + decision, + action: scenario === "normal" ? null : committed ? "confirm_external_state" : "retry_step", + mayResolve: scenario !== "authorization_denied" + } + }; +} + +export function generateRecoveryFixtureCorpus(seed = 1): readonly RecoveryPathFixture[] { + return RECOVERY_FIXTURE_SCENARIOS.map((scenario, index) => + generateRecoveryPathFixture({ scenario, seed: seed + index }) + ); +} + +export type StorageFailurePoint = "read" | "write" | "remove" | "list"; + +/** A deterministic adapter for exercising checkpoint persistence failures. */ +export function createFaultingOperationStorage( + initial: OperationRecord | undefined, + failure: StorageFailurePoint +): OperationStorage { + let value = initial; + const fault = (point: StorageFailurePoint): void => { + if (point === failure) throw new Error(`fixture_${point}_failure`); + }; + return { + read() { fault("read"); return value; }, + write(record) { fault("write"); value = record; }, + remove() { fault("remove"); value = undefined; }, + list() { fault("list"); return value ? [value] : []; } + }; +} diff --git a/docs/DETERMINISTIC_SORTING.md b/docs/DETERMINISTIC_SORTING.md new file mode 100644 index 0000000..c5a5ec9 --- /dev/null +++ b/docs/DETERMINISTIC_SORTING.md @@ -0,0 +1,21 @@ +# Deterministic mixed-status sorting + +`core/sorting` provides immutable sort helpers for operational records and +events. Record lists put actionable failures first, followed by interrupted, +pending, running, successful, skipped, and abandoned records. Within a status +they use newest timestamp, highest sequence, status text, then id. Event streams +remain newest-first and use status, sequence, and id only as tie-breakers. + +Unknown statuses sort after known statuses by their literal value. Missing or +invalid timestamps sort as the oldest value. These rules make a retry, reversed +input, or an engine with an unstable native sort produce the same result. + +Authorization must happen before sorting. The helpers are deliberately pure and +cannot grant visibility; pass only the records the actor may see. They return a +copy and never mutate source data, so failure or rollback means discarding the +result. Adding a new status is backwards compatible (it initially sorts in the +unknown group); add it to `STATUS_PRIORITY` only when its operational priority +has been agreed, and update cursor/version contracts if persisted pagination +depends on the old order. + +Verification: `npx vitest run core/sorting`. diff --git a/docs/OPERATION_RECOVERY.md b/docs/OPERATION_RECOVERY.md index 36b58aa..86e6de7 100644 --- a/docs/OPERATION_RECOVERY.md +++ b/docs/OPERATION_RECOVERY.md @@ -195,6 +195,12 @@ await runOperation(definition, { operationId: `redeem:${eventId}`, context, stor should not continue. The record is kept, so the audit trail still explains what was attempted. +At a request boundary, use `resolveInterruptedStepAuthorized(auth, ...)` and +`abandonOperationAuthorized(auth, ...)`. Both require `admin:recovery`, deny +unknown/non-maintainer roles before reading or writing a checkpoint, and leave +the record byte-for-byte unchanged on denial. The unguarded functions are kept +for trusted workers and backwards compatibility. + ## Design decisions and tradeoffs - **One entry point.** Starting and resuming are the same call, so there is no diff --git a/docs/POLICY_SIMULATION.md b/docs/POLICY_SIMULATION.md index fb144e2..961f869 100644 --- a/docs/POLICY_SIMULATION.md +++ b/docs/POLICY_SIMULATION.md @@ -60,3 +60,12 @@ The simulation engine (`core/policy/simulation.ts`) operates as a **strictly rea - **Explanation** — `outcome` is `no_impact`, `limited_impact`, or `broad_impact` (more than 30% of evaluated records affected). `examples` lists the most severe affected records with the reason for each, capped by `maxExamples` (default 5; the rest are counted in `omittedExamples`). `report.operationalWarnings` carries the risk warnings. The policy and the records are only read. Run it against fixtures or an export; it has no write path to production data. + +Retries are deterministic when callers pass `now`: the simulation identifier +is derived from that evaluation timestamp, and affected examples are ordered by +severity then record id. Keep the same snapshot and `now` while retrying after +a transport failure. A new snapshot should use a new timestamp and is a new +preview. Invalid selection ids or bounds fail before evaluation; authorization +denials fail before records are inspected. Because a preview never writes, +rollback consists only of discarding it. Applying a policy remains a separate, +authorized operation and must take its own recovery checkpoint. diff --git a/docs/RECOVERY_FIXTURES.md b/docs/RECOVERY_FIXTURES.md new file mode 100644 index 0000000..7589aa8 --- /dev/null +++ b/docs/RECOVERY_FIXTURES.md @@ -0,0 +1,22 @@ +# Rare recovery fixtures + +`core/testing/recoveryFixtures.ts` generates deterministic, JSON-safe operation +records for normal completion, failure before an external effect, retry, +ambiguous committed effects, and authorization denial. Equal seeds produce +equal records, timestamps, ids, and idempotency keys, which makes CI failures +replayable. `createFaultingOperationStorage` adds explicit read/write/remove/list +faults for persistence-edge tests. + +Use the corpus in table-driven tests, or request one scenario when a test needs +to mutate it. Unknown scenarios throw rather than silently falling back. The +generators perform no I/O and never mutate production state. + +An ambiguous-commit fixture must be resolved by an authorized maintainer; never +convert it directly to a retry, because the external effect may already exist. +Authorization-denied and checkpoint-write failures must leave the original +record unchanged. Recovery is to fix the permission/storage dependency and +re-run using the same fixture seed and idempotency key. Rollback is simply +discarding generated data; fixtures are not migration or production records. + +This is an additive testing API and requires no consumer migration. Verify it +with `npx vitest run core/testing/__tests__/recoveryFixtures.test.ts`.