diff --git a/apps/api/src/findings/__tests__/findings.relocation.spec.ts b/apps/api/src/findings/__tests__/findings.relocation.spec.ts new file mode 100644 index 00000000..abcbbf22 --- /dev/null +++ b/apps/api/src/findings/__tests__/findings.relocation.spec.ts @@ -0,0 +1,134 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { createLocationAnchor } from '@engine/core/location-mapper'; +import { findingsRepository } from '../findings.repository'; +import { FindingsService } from '../findings.service'; + +const FIXTURES = path.resolve(__dirname, '../../../../../tests/fixtures/location-mapping'); +const read = (name: string) => fs.readFileSync(path.join(FIXTURES, name), 'utf8'); +const v1 = read('runner.v1.ts'); +const v2 = read('runner.v2.ts'); +const v4 = read('runner.v4-rewritten.ts'); + +const FILE = 'src/runner.ts'; + +function seed(service: FindingsService, org: string, repo: string, line: number) { + return service.create({ + organizationId: org, + repositoryId: repo, + analysisJobId: 'job-1', + title: `exec at ${line}`, + description: 'd', + severity: 'high', + ruleId: 'cmd-001', + filePath: FILE, + line, + locationAnchor: createLocationAnchor(v1, { startLine: line }), + }); +} + +describe('finding location relocation', () => { + const service = new FindingsService(findingsRepository); + + beforeEach(() => findingsRepository.clear()); + + it('stores the anchor and marks new findings exact', () => { + const f = seed(service, 'org-a', 'repo-1', 9); + expect(f.locationAnchor?.startLine).toBe(9); + expect(f.locationStatus).toBe('exact'); + }); + + it('leaves findings created without an anchor unchanged (backward compatible)', () => { + const f = service.create({ + organizationId: 'org-a', repositoryId: 'repo-1', analysisJobId: 'j', + title: 't', description: 'd', severity: 'low', ruleId: 'r', filePath: FILE, line: 3, + }); + expect(f.locationAnchor).toBeUndefined(); + expect(f.locationStatus).toBeUndefined(); + const res = service.relocateFindings({ organizationId: 'org-a', repositoryId: 'repo-1', files: { [FILE]: v2 } }); + expect(res).toMatchObject({ examined: 1, skippedNoAnchor: 1, relocated: 0 }); + expect(service.getForTenant(f.id, 'org-a')!.line).toBe(3); + }); + + it('moves each finding to its new line without touching its fingerprint', () => { + const lookup = seed(service, 'org-a', 'repo-1', 9); + const tail = seed(service, 'org-a', 'repo-1', 14); + const fingerprints = [lookup.fingerprint, tail.fingerprint]; + + const res = service.relocateFindings({ + organizationId: 'org-a', repositoryId: 'repo-1', files: new Map([[`./${FILE}`, v2]]), + }); + + expect(res).toMatchObject({ examined: 2, relocated: 2, lost: 0, unchanged: 0 }); + const lines = v2.split('\n'); + const execLines = lines.map((l, i) => (l.trim() === 'return exec(cmd);' ? i + 1 : 0)).filter(Boolean); + expect(service.getForTenant(lookup.id, 'org-a')!.line).toBe(execLines[0]); + expect(service.getForTenant(tail.id, 'org-a')!.line).toBe(execLines[1]); + expect(service.getForTenant(lookup.id, 'org-a')!.locationStatus).toBe('moved'); + expect([lookup.fingerprint, tail.fingerprint]).toEqual(fingerprints); + }); + + it('is idempotent: a second run over the same contents changes nothing', () => { + seed(service, 'org-a', 'repo-1', 9); + const input = { organizationId: 'org-a', repositoryId: 'repo-1', files: { [FILE]: v2 } }; + service.relocateFindings(input); + expect(service.relocateFindings(input)).toMatchObject({ relocated: 0, unchanged: 1, lost: 0 }); + }); + + it('marks findings lost when their code is gone, then recovers them when it returns', () => { + const f = seed(service, 'org-a', 'repo-1', 9); + const base = { organizationId: 'org-a', repositoryId: 'repo-1' }; + + expect(service.relocateFindings({ ...base, files: { [FILE]: v4 } })).toMatchObject({ lost: 1 }); + let current = service.getForTenant(f.id, 'org-a')!; + expect(current.locationStatus).toBe('lost'); + expect(current.line).toBe(9); + + service.relocateFindings({ ...base, files: { [FILE]: v2 } }); + current = service.getForTenant(f.id, 'org-a')!; + expect(current.locationStatus).toBe('moved'); + expect(current.line).not.toBe(9); + }); + + it('marks findings in deleted files lost, but ignores files missing from a partial snapshot', () => { + const f = seed(service, 'org-a', 'repo-1', 9); + const base = { organizationId: 'org-a', repositoryId: 'repo-1' }; + + expect(service.relocateFindings({ ...base, files: {} })).toMatchObject({ skippedFileNotProvided: 1, lost: 0 }); + expect(service.getForTenant(f.id, 'org-a')!.locationStatus).toBe('exact'); + + expect(service.relocateFindings({ ...base, files: {}, deletedFiles: [FILE] })).toMatchObject({ lost: 1 }); + expect(service.getForTenant(f.id, 'org-a')!.locationStatus).toBe('lost'); + }); + + it('never touches another organization or repository', () => { + const own = seed(service, 'org-a', 'repo-1', 9); + const otherOrg = seed(service, 'org-b', 'repo-1', 9); + const otherRepo = seed(service, 'org-a', 'repo-2', 9); + + const res = service.relocateFindings({ organizationId: 'org-a', repositoryId: 'repo-1', files: { [FILE]: v2 } }); + + expect(res.examined).toBe(1); + expect(service.getForTenant(own.id, 'org-a')!.line).not.toBe(9); + expect(service.getForTenant(otherOrg.id, 'org-b')!.line).toBe(9); + expect(service.getForTenant(otherRepo.id, 'org-a')!.line).toBe(9); + }); + + it('ignores corrupt anchors and unsafe paths instead of throwing', () => { + const f = seed(service, 'org-a', 'repo-1', 9); + (f as { locationAnchor: unknown }).locationAnchor = { version: 99 }; + const res = service.relocateFindings({ + organizationId: 'org-a', repositoryId: 'repo-1', files: { '../escape.ts': v2, [FILE]: v2 }, + }); + expect(res).toMatchObject({ examined: 1, skippedNoAnchor: 1 }); + }); + + it('validates tenant scope', () => { + expect(() => service.relocateFindings({ organizationId: ' ', repositoryId: 'r', files: {} })).toThrow( + 'organizationId is required', + ); + expect(() => service.relocateFindings({ organizationId: 'o', repositoryId: '', files: {} })).toThrow( + 'repositoryId is required', + ); + }); +}); diff --git a/apps/api/src/findings/finding-location.ts b/apps/api/src/findings/finding-location.ts new file mode 100644 index 00000000..15725651 --- /dev/null +++ b/apps/api/src/findings/finding-location.ts @@ -0,0 +1,114 @@ +/** + * Finding location relocation (pure planning half). + * + * Decides, per finding, where its flagged code lives in a newer revision of the + * file. Persistence stays in FindingsService so this module has no state and can + * be reused by workers and reports. + */ + +import { + buildFileLineIndex, + createLocationAnchor, + FileLineIndex, + isLocationAnchor, + LocationMapOptions, + mapLocation, +} from '@engine/core/location-mapper'; +import { normalizeFindingPath } from '@engine/core/result-normalizer'; +import type { Finding, LocationAnchor, LocationStatus } from './finding.types'; + +export interface RelocateFindingsInput { + organizationId: string; + repositoryId: string; + /** Current contents by repository-relative path. Paths are normalized. */ + files: Record | Map; + /** Paths that no longer exist; anchored findings in them become 'lost'. */ + deletedFiles?: string[]; + options?: LocationMapOptions; +} + +export interface RelocateFindingsResult { + examined: number; + /** Anchored findings whose line and anchor did not need to change. */ + unchanged: number; + /** Findings whose line, anchor or status changed to a located state. */ + relocated: number; + /** Findings that could not be located (or whose file was deleted). */ + lost: number; + skippedNoAnchor: number; + /** File content for the finding was not part of this snapshot. */ + skippedFileNotProvided: number; +} + +export interface RelocationContext { + files: Map; + indexes: Map; + deleted: Set; + options: LocationMapOptions; +} + +export type RelocationPlan = + | { kind: 'skip'; reason: 'no-anchor' | 'file-not-provided' } + | { kind: 'apply'; changed: boolean; line: number; anchor: LocationAnchor; status: LocationStatus }; + +export function buildRelocationContext(input: RelocateFindingsInput): RelocationContext { + const files = new Map(); + const entries = input.files instanceof Map ? [...input.files.entries()] : Object.entries(input.files ?? {}); + for (const [path, content] of entries) { + const norm = normalizeFindingPath(path); + if (norm !== undefined && typeof content === 'string') files.set(norm, content); + } + const deleted = new Set(); + for (const path of input.deletedFiles ?? []) { + const norm = normalizeFindingPath(path); + if (norm !== undefined) deleted.add(norm); + } + return { files, indexes: new Map(), deleted, options: input.options ?? {} }; +} + +export function planFindingRelocation(finding: Finding, ctx: RelocationContext): RelocationPlan { + const anchor = finding.locationAnchor; + const path = normalizeFindingPath(finding.filePath); + if (!isLocationAnchor(anchor) || path === undefined) return { kind: 'skip', reason: 'no-anchor' }; + + const keep = (status: LocationStatus): RelocationPlan => ({ + kind: 'apply', + changed: finding.locationStatus !== status, + line: finding.line ?? anchor.startLine, + anchor, + status, + }); + + if (ctx.deleted.has(path)) return keep('lost'); + + const content = ctx.files.get(path); + if (content === undefined) return { kind: 'skip', reason: 'file-not-provided' }; + + let index = ctx.indexes.get(path); + if (!index) { + index = buildFileLineIndex(content); + ctx.indexes.set(path, index); + } + + const mapping = mapLocation(anchor, index, ctx.options); + // Keep the old anchor when lost so a later revision can still match it. + if (mapping.status === 'lost') return keep('lost'); + + const next = + createLocationAnchor(content, { startLine: mapping.startLine, endLine: mapping.endLine }, ctx.options) ?? + anchor; + const line = mapping.startLine; + const sameAnchor = JSON.stringify(next) === JSON.stringify(anchor); + // Re-checking an already-relocated finding finds it exactly where we left it. + // Keep the earlier moved/modified/ambiguous label so reruns are no-ops; only a + // recovery from 'lost' (or a first run) takes the new status. + const settled = + mapping.status === 'exact' && + finding.line === line && + sameAnchor && + finding.locationStatus !== undefined && + finding.locationStatus !== 'lost'; + const status = settled ? finding.locationStatus! : mapping.status; + const changed = finding.line !== line || finding.locationStatus !== status || !sameAnchor; + return { kind: 'apply', changed, line, anchor: next, status }; +} diff --git a/apps/api/src/findings/finding.types.ts b/apps/api/src/findings/finding.types.ts index af6ffa93..dd8f150e 100644 --- a/apps/api/src/findings/finding.types.ts +++ b/apps/api/src/findings/finding.types.ts @@ -2,6 +2,10 @@ * Finding domain types and list query contract (#992). */ +import type { LocationAnchor, LocationStatus } from '@engine/core/location-mapper'; + +export type { LocationAnchor, LocationStatus }; + export type FindingSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info'; export type FindingStatus = 'open' | 'suppressed' | 'resolved' | 'accepted'; @@ -18,6 +22,13 @@ export interface Finding { ruleId: string; filePath?: string; line?: number; + /** + * Content anchor for `line`, used to follow the code as the file changes. + * Absent on findings created before location mapping existed. + */ + locationAnchor?: LocationAnchor; + /** Result of the last relocation against a newer revision of the file. */ + locationStatus?: LocationStatus; /** Current assignee (user/team/module) (#1035). */ assignedTo?: string; /** Actor who performed the last assignment/reassignment (#1035). */ diff --git a/apps/api/src/findings/findings.service.ts b/apps/api/src/findings/findings.service.ts index 306ae67e..4a17ffcc 100644 --- a/apps/api/src/findings/findings.service.ts +++ b/apps/api/src/findings/findings.service.ts @@ -29,6 +29,13 @@ import { } from './finding.types'; import { findingsToCsv } from './findings.csv'; import { computeFindingFingerprint } from './finding.fingerprint'; +import { + RelocateFindingsInput, + RelocateFindingsResult, + planFindingRelocation, + buildRelocationContext, +} from './finding-location'; +import type { LocationAnchor } from './finding.types'; export interface CreateFindingInput { organizationId: string; @@ -41,6 +48,8 @@ export interface CreateFindingInput { ruleId: string; filePath?: string; line?: number; + /** Content anchor captured at `line`; enables relocation across revisions. */ + locationAnchor?: LocationAnchor; expiresAt?: string; } @@ -128,6 +137,9 @@ export class FindingsService { ruleId: input.ruleId, filePath: input.filePath, line: input.line, + ...(input.locationAnchor + ? { locationAnchor: input.locationAnchor, locationStatus: 'exact' as const } + : {}), expiresAt: input.expiresAt, createdAt: now, updatedAt: now, @@ -165,6 +177,66 @@ export class FindingsService { return this.repo.list(query); } + /** + * Re-point anchored findings of one repository at their new lines after the + * files changed. Scoped to (organizationId, repositoryId); idempotent for the + * same file contents. Files absent from `files` are left untouched, so a + * partial snapshot never marks findings as lost. + */ + relocateFindings(input: RelocateFindingsInput): RelocateFindingsResult { + for (const field of ['organizationId', 'repositoryId'] as const) { + if (typeof input?.[field] !== 'string' || !input[field].trim()) { + throw Object.assign(new Error(`${field} is required`), { + code: 'VALIDATION_ERROR', + status: 400, + }); + } + } + const ctx = buildRelocationContext(input); + const targets: Finding[] = []; + let cursor: string | undefined; + do { + const page = this.repo.list({ + organizationId: input.organizationId, + repositoryId: input.repositoryId, + limit: MAX_PAGE_LIMIT, + cursor, + }); + targets.push(...page.items); + cursor = page.nextCursor ?? undefined; + } while (cursor); + + const result: RelocateFindingsResult = { + examined: targets.length, + unchanged: 0, + relocated: 0, + lost: 0, + skippedNoAnchor: 0, + skippedFileNotProvided: 0, + }; + const now = new Date().toISOString(); + for (const finding of targets) { + const plan = planFindingRelocation(finding, ctx); + if (plan.kind === 'skip') { + if (plan.reason === 'no-anchor') result.skippedNoAnchor++; + else result.skippedFileNotProvided++; + continue; + } + if (!plan.changed) { + result.unchanged++; + continue; + } + finding.line = plan.line; + finding.locationAnchor = plan.anchor; + finding.locationStatus = plan.status; + finding.updatedAt = now; + this.repo.upsert(finding); + if (plan.status === 'lost') result.lost++; + else result.relocated++; + } + return result; + } + getForTenant(id: string, organizationId: string): Finding | undefined { return this.repo.getForTenant(id, organizationId); } diff --git a/apps/api/src/findings/index.ts b/apps/api/src/findings/index.ts index ca2fb861..90197cc4 100644 --- a/apps/api/src/findings/index.ts +++ b/apps/api/src/findings/index.ts @@ -9,3 +9,6 @@ export * from './findings.routes'; // Symbol-aware finding context (#1164) export * from './symbol-context'; + +// Stable location mapping +export * from './finding-location'; diff --git a/apps/api/src/queue/priority-queue.spec.ts b/apps/api/src/queue/priority-queue.spec.ts new file mode 100644 index 00000000..e7047559 --- /dev/null +++ b/apps/api/src/queue/priority-queue.spec.ts @@ -0,0 +1,323 @@ +import { + DEFAULT_PRIORITY_QUEUE_CONFIG, + JOB_PRIORITIES, + JobPriority, + JobPriorityQueue, + isJobPriority, + resolveJobPriority, + toBullMqPriority, +} from "./priority-queue"; + +function drain(queue: JobPriorityQueue, now: number): string[] { + const out: string[] = []; + for (let job = queue.dequeue(now); job; job = queue.dequeue(now)) { + out.push(job.id); + } + return out; +} + +describe("JobPriorityQueue ordering", () => { + it("runs higher priorities first regardless of arrival order", () => { + const queue = new JobPriorityQueue(); + queue.enqueue("low-1", "x", { priority: "low" }, 0); + queue.enqueue("normal-1", "x", { priority: "normal" }, 0); + queue.enqueue("critical-1", "x", { priority: "critical" }, 0); + queue.enqueue("high-1", "x", { priority: "high" }, 0); + + expect(drain(queue, 0)).toEqual(["critical-1", "high-1", "normal-1", "low-1"]); + }); + + it("is first-in first-out within a priority", () => { + const queue = new JobPriorityQueue(); + for (let i = 0; i < 5; i += 1) queue.enqueue(`job-${i}`, i, { priority: "high" }, i); + + expect(drain(queue, 5)).toEqual(["job-0", "job-1", "job-2", "job-3", "job-4"]); + }); + + it("defaults to normal priority", () => { + const queue = new JobPriorityQueue(); + expect(queue.enqueue("a", "x", {}, 0)).toEqual({ accepted: true, priority: "normal" }); + expect(queue.enqueue("b", "x", undefined, 0)).toEqual({ accepted: true, priority: "normal" }); + queue.enqueue("c", "x", { priority: "high" }, 0); + expect(drain(queue, 0)).toEqual(["c", "a", "b"]); + }); + + it("returns the payload and timing, and undefined when empty", () => { + const queue = new JobPriorityQueue<{ repo: string }>(); + expect(queue.dequeue(0)).toBeUndefined(); + expect(queue.peek(0)).toBeUndefined(); + + queue.enqueue("a", { repo: "r1" }, { priority: "low" }, 1_000); + expect(queue.dequeue(1_500)).toEqual({ + id: "a", + payload: { repo: "r1" }, + priority: "low", + effectivePriority: "low", + enqueuedAt: 1_000, + waitedMs: 500, + }); + expect(queue.dequeue(1_500)).toBeUndefined(); + }); + + it("peek does not remove or reorder anything", () => { + const queue = new JobPriorityQueue(); + queue.enqueue("a", "x", { priority: "low" }, 0); + queue.enqueue("b", "x", { priority: "high" }, 0); + + expect(queue.peek(0)?.id).toBe("b"); + expect(queue.peek(0)?.id).toBe("b"); + expect(queue.size()).toBe(2); + expect(drain(queue, 0)).toEqual(["b", "a"]); + }); + + it("orders deterministically for a large mixed workload", () => { + const queue = new JobPriorityQueue({ agingMs: Number.MAX_SAFE_INTEGER }); + const order: JobPriority[] = ["low", "high", "normal", "critical"]; + for (let i = 0; i < 1_000; i += 1) { + queue.enqueue(`job-${i}`, i, { priority: order[i % 4] }, i); + } + + const out = drain(queue, 1_000); + expect(out).toHaveLength(1_000); + // critical (i % 4 === 3) first, then high (1), normal (2), low (0), each in arrival order + const expected = [3, 1, 2, 0].flatMap((m) => + Array.from({ length: 250 }, (_, k) => `job-${k * 4 + m}`), + ); + expect(out).toEqual(expected); + }); +}); + +describe("JobPriorityQueue aging", () => { + const config = { agingMs: 1_000 }; + + it("does not promote before the aging interval and promotes exactly at it", () => { + const queue = new JobPriorityQueue(config); + queue.enqueue("old-normal", "x", { priority: "normal" }, 0); + queue.enqueue("new-high", "x", { priority: "high" }, 999); + + // 999ms old: still normal, so high goes first. + expect(queue.peek(999)?.id).toBe("new-high"); + // 1000ms old: promoted to high; it arrived first so it now goes first. + const next = queue.peek(1_000); + expect(next).toMatchObject({ id: "old-normal", priority: "normal", effectivePriority: "high" }); + }); + + it("never promotes above critical", () => { + const queue = new JobPriorityQueue(config); + queue.enqueue("low", "x", { priority: "low" }, 0); + expect(queue.peek(1_000_000)?.effectivePriority).toBe("critical"); + }); + + it("lets an old low-priority job overtake newer critical work, so it cannot starve", () => { + const queue = new JobPriorityQueue(config); + queue.enqueue("old-low", "x", { priority: "low" }, 0); + // A steady stream of critical jobs arrives after the low job has aged to critical. + queue.enqueue("c1", "x", { priority: "critical" }, 3_000); + queue.enqueue("c2", "x", { priority: "critical" }, 3_001); + + expect(drain(queue, 3_001)).toEqual(["old-low", "c1", "c2"]); + }); + + it("serves a continuous critical stream without starving low work for long", () => { + const queue = new JobPriorityQueue(config); + queue.enqueue("low", "x", { priority: "low" }, 0); + let servedAt: number | undefined; + for (let t = 0; t <= 10_000 && servedAt === undefined; t += 100) { + queue.enqueue(`c-${t}`, "x", { priority: "critical" }, t); + const job = queue.dequeue(t); + if (job?.id === "low") servedAt = t; + } + // Low is promoted to critical after 3 x agingMs and then wins on arrival order. + expect(servedAt).toBeDefined(); + expect(servedAt as number).toBeLessThanOrEqual(3_000); + }); + + it("counts jobs that left ahead of their original level", () => { + const queue = new JobPriorityQueue(config); + queue.enqueue("a", "x", { priority: "low" }, 0); + queue.enqueue("b", "x", { priority: "low" }, 5_000); + queue.dequeue(5_000); // a has aged to critical + queue.dequeue(5_000); // b is brand new, still low + expect(queue.stats().promoted).toBe(1); + }); + + it("treats a clock that moves backwards as zero wait", () => { + const queue = new JobPriorityQueue(config); + queue.enqueue("a", "x", { priority: "normal" }, 5_000); + expect(queue.peek(1_000)).toMatchObject({ waitedMs: 0, effectivePriority: "normal" }); + }); +}); + +describe("JobPriorityQueue capacity and duplicates", () => { + it("applies backpressure per priority without blocking other levels", () => { + const queue = new JobPriorityQueue({ maxSizePerPriority: 2 }); + expect(queue.enqueue("l1", "x", { priority: "low" }, 0).accepted).toBe(true); + expect(queue.enqueue("l2", "x", { priority: "low" }, 0).accepted).toBe(true); + expect(queue.enqueue("l3", "x", { priority: "low" }, 0)).toEqual({ accepted: false, reason: "queue_full" }); + + // A full low level cannot block a critical job. + expect(queue.enqueue("c1", "x", { priority: "critical" }, 0).accepted).toBe(true); + expect(queue.sizeByPriority()).toEqual({ critical: 1, high: 0, normal: 0, low: 2 }); + expect(queue.stats().rejectedFull).toBe(1); + }); + + it("frees capacity when jobs are dequeued or removed", () => { + const queue = new JobPriorityQueue({ maxSizePerPriority: 1 }); + queue.enqueue("a", "x", { priority: "high" }, 0); + expect(queue.enqueue("b", "x", { priority: "high" }, 0).accepted).toBe(false); + + queue.dequeue(0); + expect(queue.enqueue("b", "x", { priority: "high" }, 0).accepted).toBe(true); + queue.remove("b"); + expect(queue.enqueue("c", "x", { priority: "high" }, 0).accepted).toBe(true); + }); + + it("rejects a duplicate id while queued but allows it again after it leaves", () => { + const queue = new JobPriorityQueue(); + queue.enqueue("a", "x", { priority: "low" }, 0); + expect(queue.enqueue("a", "y", { priority: "critical" }, 0)).toEqual({ + accepted: false, + reason: "duplicate", + }); + expect(queue.size()).toBe(1); + expect(queue.dequeue(0)?.payload).toBe("x"); + expect(queue.enqueue("a", "z", {}, 0).accepted).toBe(true); + expect(queue.stats().rejectedDuplicate).toBe(1); + }); +}); + +describe("JobPriorityQueue removal and introspection", () => { + it("removes a queued job from anywhere in the queue", () => { + const queue = new JobPriorityQueue(); + queue.enqueue("a", "x", { priority: "high" }, 0); + queue.enqueue("b", "x", { priority: "high" }, 0); + queue.enqueue("c", "x", { priority: "high" }, 0); + + expect(queue.remove("b")).toBe(true); + expect(queue.remove("b")).toBe(false); + expect(queue.remove("missing")).toBe(false); + expect(queue.has("b")).toBe(false); + expect(queue.size()).toBe(2); + expect(drain(queue, 0)).toEqual(["a", "c"]); + }); + + it("skips a removed head and reports the next oldest enqueue time", () => { + const queue = new JobPriorityQueue(); + expect(queue.oldestEnqueuedAt()).toBeUndefined(); + + queue.enqueue("a", "x", { priority: "low" }, 100); + queue.enqueue("b", "x", { priority: "critical" }, 200); + queue.enqueue("c", "x", { priority: "low" }, 300); + expect(queue.oldestEnqueuedAt()).toBe(100); + + queue.remove("a"); + expect(queue.oldestEnqueuedAt()).toBe(200); + queue.dequeue(300); + expect(queue.oldestEnqueuedAt()).toBe(300); + }); + + it("stays correct and bounded through heavy enqueue/remove churn", () => { + const queue = new JobPriorityQueue({ maxSizePerPriority: 10 }); + queue.enqueue("head", 0, { priority: "normal" }, 0); + for (let i = 0; i < 5_000; i += 1) { + expect(queue.enqueue(`tmp-${i}`, i, { priority: "normal" }, i).accepted).toBe(true); + expect(queue.remove(`tmp-${i}`)).toBe(true); + } + expect(queue.size()).toBe(1); + expect(drain(queue, 6_000)).toEqual(["head"]); + }); + + it("compacts after many dequeues without losing order", () => { + const queue = new JobPriorityQueue(); + for (let i = 0; i < 300; i += 1) queue.enqueue(`j-${i}`, i, { priority: "low" }, i); + for (let i = 0; i < 200; i += 1) expect(queue.dequeue(300)?.id).toBe(`j-${i}`); + queue.enqueue("late", 0, { priority: "low" }, 301); + expect(drain(queue, 301)).toEqual([...Array.from({ length: 100 }, (_, k) => `j-${k + 200}`), "late"]); + }); + + it("tracks stats", () => { + const queue = new JobPriorityQueue(); + queue.enqueue("a", "x", {}, 0); + queue.enqueue("b", "x", {}, 0); + queue.remove("b"); + queue.dequeue(0); + expect(queue.stats()).toEqual({ + enqueued: 2, + dequeued: 1, + removed: 1, + rejectedDuplicate: 0, + rejectedFull: 0, + promoted: 0, + }); + }); +}); + +describe("JobPriorityQueue input validation", () => { + it("rejects invalid ids, priorities and clock values", () => { + const queue = new JobPriorityQueue(); + expect(() => queue.enqueue("", "x", {}, 0)).toThrow(TypeError); + expect(() => queue.enqueue(" ", "x", {}, 0)).toThrow(TypeError); + expect(() => queue.enqueue(42 as unknown as string, "x", {}, 0)).toThrow(TypeError); + expect(() => queue.enqueue("a".repeat(257), "x", {}, 0)).toThrow(RangeError); + expect(() => queue.enqueue("a", "x", { priority: "urgent" as JobPriority }, 0)).toThrow(TypeError); + expect(() => queue.enqueue("a", "x", {}, Number.NaN)).toThrow(TypeError); + expect(() => queue.dequeue(Number.POSITIVE_INFINITY)).toThrow(TypeError); + expect(() => queue.peek("now" as unknown as number)).toThrow(TypeError); + expect(queue.size()).toBe(0); + }); + + it("accepts an id at the length limit", () => { + expect(new JobPriorityQueue().enqueue("a".repeat(256), "x", {}, 0).accepted).toBe(true); + }); + + it("rejects invalid configuration", () => { + expect(() => new JobPriorityQueue({ maxSizePerPriority: 0 })).toThrow(RangeError); + expect(() => new JobPriorityQueue({ maxSizePerPriority: 1.5 })).toThrow(RangeError); + expect(() => new JobPriorityQueue({ agingMs: 0 })).toThrow(RangeError); + expect(() => new JobPriorityQueue({ agingMs: -1 })).toThrow(RangeError); + }); + + it("has safe defaults", () => { + expect(DEFAULT_PRIORITY_QUEUE_CONFIG).toEqual({ maxSizePerPriority: 10_000, agingMs: 30_000 }); + }); +}); + +describe("resolveJobPriority", () => { + it("defaults missing requests to normal", () => { + expect(resolveJobPriority(undefined)).toBe("normal"); + expect(resolveJobPriority(null)).toBe("normal"); + }); + + it("passes through allowed priorities", () => { + expect(resolveJobPriority("high")).toBe("high"); + expect(resolveJobPriority("normal")).toBe("normal"); + expect(resolveJobPriority("low")).toBe("low"); + }); + + it("keeps critical reserved: requests above the ceiling are lowered to it", () => { + expect(resolveJobPriority("critical")).toBe("high"); + expect(resolveJobPriority("critical", "critical")).toBe("critical"); + expect(resolveJobPriority("high", "normal")).toBe("normal"); + expect(resolveJobPriority(undefined, "low")).toBe("low"); + }); + + it("throws on unrecognised values instead of guessing", () => { + expect(() => resolveJobPriority("urgent")).toThrow(TypeError); + expect(() => resolveJobPriority(1)).toThrow(TypeError); + expect(() => resolveJobPriority("")).toThrow(TypeError); + expect(() => resolveJobPriority("high", "urgent" as JobPriority)).toThrow(TypeError); + }); +}); + +describe("BullMQ priority mapping", () => { + it("maps to BullMQ's scale where 1 is highest", () => { + expect(JOB_PRIORITIES.map(toBullMqPriority)).toEqual([1, 2, 3, 4]); + expect(() => toBullMqPriority("urgent" as JobPriority)).toThrow(TypeError); + }); + + it("recognises valid priorities only", () => { + expect(isJobPriority("critical")).toBe(true); + expect(isJobPriority("Critical")).toBe(false); + expect(isJobPriority(undefined)).toBe(false); + }); +}); diff --git a/apps/api/src/queue/priority-queue.ts b/apps/api/src/queue/priority-queue.ts new file mode 100644 index 00000000..3d7d52e7 --- /dev/null +++ b/apps/api/src/queue/priority-queue.ts @@ -0,0 +1,354 @@ +/** + * Job priority queue. + * + * Scans are not equally urgent: a blocking CI check should not wait behind a + * nightly full-repository audit. The queue orders work by priority and keeps + * lower priorities from starving through aging. + * + * - Four levels: critical, high, normal (default), low. + * - Within a level, jobs leave in the order they arrived (FIFO). + * - Aging: every `agingMs` a job has waited promotes it one level. Its + * effective level never goes above critical, and a promoted job competes + * with the other jobs at that level by arrival order. A low job therefore + * waits at most about 3 x agingMs before it is ahead of any newer job. + * - Each level has its own capacity, so a flood of low-priority work can + * never block a critical job from being accepted (backpressure is per level). + * - `resolveJobPriority` clamps caller-requested priorities to a ceiling, + * so critical stays reserved for internal callers by default. + * + * The class performs no I/O and never reads the clock itself; callers pass + * `now`, so ordering is deterministic and testable. + */ + +export const JOB_PRIORITIES = ["critical", "high", "normal", "low"] as const; +export type JobPriority = (typeof JOB_PRIORITIES)[number]; + +export const DEFAULT_JOB_PRIORITY: JobPriority = "normal"; + +/** Highest priority an untrusted caller may request unless told otherwise. */ +export const DEFAULT_PRIORITY_CEILING: JobPriority = "high"; + +const MAX_ID_LENGTH = 256; + +const RANK: Record = { + critical: 0, + high: 1, + normal: 2, + low: 3, +}; + +export interface PriorityQueueConfig { + /** Maximum queued jobs per priority level. */ + maxSizePerPriority: number; + /** Waiting time after which a job is promoted one level. */ + agingMs: number; +} + +export const DEFAULT_PRIORITY_QUEUE_CONFIG: PriorityQueueConfig = { + maxSizePerPriority: 10_000, + agingMs: 30_000, +}; + +export interface EnqueueOptions { + priority?: JobPriority; +} + +export type EnqueueResult = + | { accepted: true; priority: JobPriority } + | { accepted: false; reason: "duplicate" | "queue_full" }; + +export interface QueuedJob { + id: string; + payload: T; + /** Priority the job was enqueued with. */ + priority: JobPriority; + /** Priority it is competing at now, after aging. */ + effectivePriority: JobPriority; + enqueuedAt: number; + waitedMs: number; +} + +export interface PriorityQueueStats { + enqueued: number; + dequeued: number; + removed: number; + rejectedDuplicate: number; + rejectedFull: number; + /** Jobs that left the queue ahead of their original level because of aging. */ + promoted: number; +} + +interface Entry { + id: string; + payload: T; + rank: number; + enqueuedAt: number; + seq: number; + removed: boolean; +} + +interface Level { + items: Entry[]; + head: number; + live: number; +} + +/** Compact a level's backing array once at least half of it is consumed. */ +const COMPACT_MIN_HEAD = 64; + +export class JobPriorityQueue { + private readonly config: PriorityQueueConfig; + private readonly levels: Level[] = JOB_PRIORITIES.map(() => ({ items: [], head: 0, live: 0 })); + private readonly byId: Map> = new Map(); + private seq = 0; + private readonly counters: PriorityQueueStats = { + enqueued: 0, + dequeued: 0, + removed: 0, + rejectedDuplicate: 0, + rejectedFull: 0, + promoted: 0, + }; + + constructor(config: Partial = {}) { + this.config = { ...DEFAULT_PRIORITY_QUEUE_CONFIG, ...config }; + requirePositiveInteger(this.config.maxSizePerPriority, "maxSizePerPriority"); + requirePositiveInteger(this.config.agingMs, "agingMs"); + } + + /** + * Add a job. Returns a rejection instead of throwing when the queue cannot + * take it (duplicate id, or the level is full) so callers can apply backpressure. + * Invalid arguments throw. + */ + enqueue( + id: string, + payload: T, + options: EnqueueOptions = {}, + now: number = Date.now(), + ): EnqueueResult { + requireId(id); + requireFinite(now, "now"); + const priority = options.priority ?? DEFAULT_JOB_PRIORITY; + requirePriority(priority); + + if (this.byId.has(id)) { + this.counters.rejectedDuplicate += 1; + return { accepted: false, reason: "duplicate" }; + } + const level = this.levels[RANK[priority]]; + if (level.live >= this.config.maxSizePerPriority) { + this.counters.rejectedFull += 1; + return { accepted: false, reason: "queue_full" }; + } + + const entry: Entry = { + id, + payload, + rank: RANK[priority], + enqueuedAt: now, + seq: this.seq++, + removed: false, + }; + level.items.push(entry); + level.live += 1; + this.byId.set(id, entry); + this.counters.enqueued += 1; + return { accepted: true, priority }; + } + + /** Remove and return the next job to run, or undefined when empty. */ + dequeue(now: number = Date.now()): QueuedJob | undefined { + requireFinite(now, "now"); + const entry = this.next(now); + if (!entry) { + return undefined; + } + const level = this.levels[entry.rank]; + level.head += 1; + level.live -= 1; + this.byId.delete(entry.id); + this.compact(level); + + this.counters.dequeued += 1; + const view = this.view(entry, now); + if (RANK[view.effectivePriority] < entry.rank) { + this.counters.promoted += 1; + } + return view; + } + + /** The job `dequeue` would return, without removing it. */ + peek(now: number = Date.now()): QueuedJob | undefined { + requireFinite(now, "now"); + const entry = this.next(now); + return entry ? this.view(entry, now) : undefined; + } + + /** Cancel a queued job. Returns whether it was queued. */ + remove(id: string): boolean { + const entry = this.byId.get(id); + if (!entry) { + return false; + } + entry.removed = true; + this.byId.delete(id); + const level = this.levels[entry.rank]; + level.live -= 1; + this.counters.removed += 1; + this.purge(level); + return true; + } + + has(id: string): boolean { + return this.byId.has(id); + } + + size(): number { + return this.byId.size; + } + + sizeByPriority(): Record { + const sizes = {} as Record; + for (const priority of JOB_PRIORITIES) { + sizes[priority] = this.levels[RANK[priority]].live; + } + return sizes; + } + + /** + * Enqueue time of the longest-waiting job, for `QueueAgeSlo.evaluate`. + * Undefined when the queue is empty. + */ + oldestEnqueuedAt(): number | undefined { + let oldest: number | undefined; + for (const level of this.levels) { + const head = this.headOf(level); + if (head && (oldest === undefined || head.enqueuedAt < oldest)) { + oldest = head.enqueuedAt; + } + } + return oldest; + } + + stats(): PriorityQueueStats { + return { ...this.counters }; + } + + /** + * The entry to run next: lowest effective level wins, and entries at the same + * effective level go in arrival order. Only each level's head is considered, + * because within a level the head is the oldest and so the most aged. + */ + private next(now: number): Entry | undefined { + let best: Entry | undefined; + let bestRank = Infinity; + for (const level of this.levels) { + const head = this.headOf(level); + if (!head) { + continue; + } + const rank = this.effectiveRank(head, now); + if (rank < bestRank || (rank === bestRank && best !== undefined && head.seq < best.seq)) { + best = head; + bestRank = rank; + } + } + return best; + } + + private headOf(level: Level): Entry | undefined { + while (level.head < level.items.length && level.items[level.head].removed) { + level.head += 1; + } + return level.items[level.head]; + } + + private compact(level: Level): void { + if (level.head >= COMPACT_MIN_HEAD && level.head * 2 >= level.items.length) { + level.items.splice(0, level.head); + level.head = 0; + } + } + + /** Drop cancelled entries once they outnumber live ones, bounding memory under churn. */ + private purge(level: Level): void { + const pending = level.items.length - level.head; + if (pending > 2 * level.live + COMPACT_MIN_HEAD) { + level.items = level.items.slice(level.head).filter((entry) => !entry.removed); + level.head = 0; + } + } + + private effectiveRank(entry: Entry, now: number): number { + const waited = Math.max(0, now - entry.enqueuedAt); + return Math.max(0, entry.rank - Math.floor(waited / this.config.agingMs)); + } + + private view(entry: Entry, now: number): QueuedJob { + return { + id: entry.id, + payload: entry.payload, + priority: JOB_PRIORITIES[entry.rank], + effectivePriority: JOB_PRIORITIES[this.effectiveRank(entry, now)], + enqueuedAt: entry.enqueuedAt, + waitedMs: Math.max(0, now - entry.enqueuedAt), + }; + } +} + +export function isJobPriority(value: unknown): value is JobPriority { + return typeof value === "string" && (JOB_PRIORITIES as readonly string[]).includes(value); +} + +/** + * Priority to use for a request. Missing means the default; an unrecognised + * value is an error (never silently promoted); anything above `ceiling` is + * lowered to it. + */ +export function resolveJobPriority( + requested: unknown, + ceiling: JobPriority = DEFAULT_PRIORITY_CEILING, +): JobPriority { + requirePriority(ceiling); + if (requested === undefined || requested === null) { + return RANK[DEFAULT_JOB_PRIORITY] < RANK[ceiling] ? ceiling : DEFAULT_JOB_PRIORITY; + } + if (!isJobPriority(requested)) { + throw new TypeError(`priority must be one of: ${JOB_PRIORITIES.join(", ")}`); + } + return RANK[requested] < RANK[ceiling] ? ceiling : requested; +} + +/** BullMQ job priority: 1 is the highest, larger numbers run later. */ +export function toBullMqPriority(priority: JobPriority): number { + requirePriority(priority); + return RANK[priority] + 1; +} + +function requirePriority(priority: unknown): asserts priority is JobPriority { + if (!isJobPriority(priority)) { + throw new TypeError(`priority must be one of: ${JOB_PRIORITIES.join(", ")}`); + } +} + +function requireId(id: unknown): void { + if (typeof id !== "string" || id.trim() === "") { + throw new TypeError("id must be a non-empty string"); + } + if (id.length > MAX_ID_LENGTH) { + throw new RangeError(`id must be at most ${MAX_ID_LENGTH} characters`); + } +} + +function requireFinite(value: unknown, field: string): void { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new TypeError(`${field} must be a finite number`); + } +} + +function requirePositiveInteger(value: number, field: string): void { + if (!Number.isInteger(value) || value <= 0) { + throw new RangeError(`${field} must be a positive integer`); + } +} diff --git a/apps/api/src/scanner/command-injection-scan.job.ts b/apps/api/src/scanner/command-injection-scan.job.ts index 57ffa47c..bf1a94d6 100644 --- a/apps/api/src/scanner/command-injection-scan.job.ts +++ b/apps/api/src/scanner/command-injection-scan.job.ts @@ -6,7 +6,9 @@ */ import { runAnalyzerSafely } from '@engine/core/analyzer-runner'; -import type { AnalyzerConfig, Finding as EngineFinding } from '@engine/core/analyzer-interface'; +import type { AnalyzerConfig } from '@engine/core/analyzer-interface'; +import { normalizeAnalysisResult, NormalizedFinding, normalizeFindingPath } from '@engine/core/result-normalizer'; +import { createLocationAnchor } from '@engine/core/location-mapper'; import { CommandInjectionAnalysisMetrics, CommandInjectionAnalysisMetricsSnapshot, @@ -79,6 +81,8 @@ export interface CommandInjectionScanOutcome { findingsAlreadyPersisted: number; findingsByRule: Record; fileErrors: Array<{ file: string; message: string }>; + /** Findings dropped or repaired by result normalization (see docs/ANALYZER_RESULT_NORMALIZATION.md). */ + normalization: { droppedFindings: number; duplicateFindings: number; issues: number }; durationMs: number; } @@ -166,10 +170,10 @@ export class CommandInjectionScanJobHandler { ); } - const result = outcome.result; + const result = normalizeAnalysisResult(outcome.result, { analyzerName: analyzer.getName() }); let persisted: { created: number; existing: number }; try { - persisted = this.persist(payload, analyzer, result.findings); + persisted = this.persist(payload, analyzer, result.findings, files); } catch { this.logger.warn('command_injection_scan.persistence_failed', { ...logFields, @@ -181,7 +185,7 @@ export class CommandInjectionScanJobHandler { const snapshot: CommandInjectionAnalysisMetricsSnapshot = telemetry.snapshot(); const findingsByRule: Record = {}; for (const f of result.findings) findingsByRule[f.ruleId] = (findingsByRule[f.ruleId] ?? 0) + 1; - const fileErrors = (result.errors ?? []).map(({ file, message }) => ({ file, message })); + const fileErrors = result.errors; const durationMs = this.now() - started; this.metrics?.recordAnalyzerDuration(analyzer.getName(), durationMs); @@ -200,6 +204,11 @@ export class CommandInjectionScanJobHandler { findingsAlreadyPersisted: persisted.existing, findingsByRule, fileErrors, + normalization: { + droppedFindings: result.counts.dropped, + duplicateFindings: result.counts.duplicates, + issues: result.counts.issues, + }, durationMs, }; @@ -212,6 +221,7 @@ export class CommandInjectionScanJobHandler { filesSkipped: scanOutcome.filesSkipped, skippedByReason: snapshot.skippedByReason, fileErrorCount: fileErrors.length, + normalization: scanOutcome.normalization, findingsDetected: scanOutcome.findingsDetected, findingsPersisted: scanOutcome.findingsPersisted, findingsAlreadyPersisted: scanOutcome.findingsAlreadyPersisted, @@ -225,7 +235,8 @@ export class CommandInjectionScanJobHandler { private persist( payload: CommandInjectionScanJobPayload, analyzer: CommandInjectionAnalyzer, - engineFindings: EngineFinding[], + engineFindings: NormalizedFinding[], + files: Map, ): { created: number; existing: number } { const { organizationId, repositoryId, analysisJobId } = payload; const existingKeys = new Set(); @@ -246,6 +257,12 @@ export class CommandInjectionScanJobHandler { cursor = page.nextCursor ?? undefined; } while (cursor); + const contentByPath = new Map(); + for (const [p, content] of files) { + const norm = normalizeFindingPath(p); + if (norm !== undefined) contentByPath.set(norm, content); + } + let created = 0; let existing = 0; for (const f of engineFindings) { @@ -265,6 +282,10 @@ export class CommandInjectionScanJobHandler { ruleId: f.ruleId, filePath: f.location.file, line: f.location.startLine, + locationAnchor: createLocationAnchor(contentByPath.get(f.location.file) ?? '', { + startLine: f.location.startLine, + endLine: f.location.endLine, + }), }); existingKeys.add(key); created++; diff --git a/docs/ANALYZER_RESULT_NORMALIZATION.md b/docs/ANALYZER_RESULT_NORMALIZATION.md new file mode 100644 index 00000000..c9d43955 --- /dev/null +++ b/docs/ANALYZER_RESULT_NORMALIZATION.md @@ -0,0 +1,85 @@ +# Analyzer Result Normalization + +Analyzers report findings in slightly different shapes: backslash paths, `./` prefixes, severity +spellings such as `ERROR` or `warning`, inverted or zero-based line ranges, flat `file`/`line` +fields, and duplicate reports. Normalization converts any analyzer output into one canonical, +deterministic shape before it reaches persistence or reporting. + +| Layer | Location | +| --- | --- | +| Normalizer (pure, engine) | `libs/engine/core/result-normalizer.ts` | +| Worker integration | `apps/api/src/scanner/command-injection-scan.job.ts` | +| Fixtures | `tests/fixtures/result-normalization/` | +| Tests | `libs/engine/core/result-normalizer.spec.ts` | + +## Public interface + +```ts +import { normalizeAnalysisResult, normalizeFindings } from '@engine/core/result-normalizer'; + +const normalized = normalizeAnalysisResult(analyzerResult, { + analyzerName: 'command-injection', + rootDir: '/workspace/job-123', // optional +}); +``` + +- `normalizeAnalysisResult(raw, options)` returns a `NormalizedAnalysisResult` + (`schemaVersion`, `findings`, `summary`, `errors`, `issues`, `counts`, `telemetry`). +- `normalizeFindings(rawFindings, options)` normalizes only a findings array. +- `normalizeFindingPath` and `normalizeSeverity` are exported for reuse. + +## Rules + +| Field | Behavior | +| --- | --- | +| `ruleId` | Trimmed. Missing or empty: finding dropped (`MISSING_RULE_ID`). | +| `location.file` | Repository-relative, `/` separators, `.`/empty segments removed, `..` resolved. Traversal above the root, absolute paths outside `rootDir`, empty paths and control characters: finding dropped (`INVALID_PATH`). Host paths never reach stored results. | +| `severity` | Case-insensitive, with aliases (`error`→high, `warning`/`warn`→medium, `note`/`hint`→info, …). Unknown values use `fallbackSeverity` (default `info`) and report `UNKNOWN_SEVERITY`. | +| lines | Integers `>= 1`. Fractions are floored. An invalid `startLine` becomes `1` (`INVALID_LINE`). `endLine < startLine` becomes `startLine`. Legacy `location.line` and flat `file`/`line` are accepted. | +| columns | Invalid values are ignored (`INVALID_COLUMN`). `endColumn` is dropped when there is no `startColumn`, and clamped to `startColumn` on a single line. | +| `message` | CRLF becomes LF, trimmed, truncated to `maxMessageLength` (default 2000). Empty: the `ruleId` is used (`EMPTY_MESSAGE`). | +| `estimatedGasSavings` | Must be a finite number `>= 0`, otherwise removed (`INVALID_GAS_SAVINGS`). | +| `metadata` | Copied as JSON-safe data with sorted keys. Functions, `undefined` and non-finite numbers are removed. | +| duplicates | Same rule, location and message produce one finding (highest severity wins). Counted in `counts.duplicates`. | +| `summary`, `totalEstimatedGasSavings` | Recomputed from the normalized findings, never trusted from the analyzer. | + +## Determinism + +Findings are sorted by file, start line, start column, rule id, severity (most severe first), then +key. Each finding gets a `key`: the first 32 hex chars of a sha256 over rule, location and message. +The same findings in any order, with any timing, give byte-identical output. Run-dependent values +(`analysisTime`, `memoryUsage`, `incremental`) are moved to `telemetry` and are excluded from this +guarantee. Normalizing an already normalized result changes nothing. + +## Failure semantics + +- The normalizer is **pure and never throws**. It does no I/O, uses no clock and keeps no shared + state, so it is safe per tenant and safe to retry. +- **Unsupported input**: a non-object result gives an empty result with a `MALFORMED_RESULT` issue. + A non-array `findings` field is treated the same way. +- **Partial failure**: a bad finding is dropped (`issues[].dropped === true`) or repaired + (`dropped === false`). Other findings are unaffected. Callers should log `counts` and decide + whether `counts.dropped > 0` should mark a scan `partial`. +- **Issue volume**: `issues` is capped at `maxIssues` (default 100). `counts.issues` is always exact. +- **Retries**: because output depends only on input and options, a retried job produces the same + findings and the same `key`s, so persistence stays idempotent. + +## Configuration + +| Option | Default | Purpose | +| --- | --- | --- | +| `fallbackSeverity` | `info` | Severity for unknown or missing values. | +| `maxMessageLength` | `2000` | Message truncation limit. | +| `rootDir` | none | Strip this absolute prefix from absolute paths. Anything outside it is rejected. | +| `maxIssues` | `100` | Limit on recorded issues. | +| `analyzerName` | none | Carried through to the result (`normalizeAnalysisResult` only). | + +## Backward compatibility + +- `schemaVersion` is `1`. Additive fields will not bump it. Removing or renaming fields will. +- Accepted legacy input shapes: `location.line`, flat `file`/`filePath`/`line`, and the severity + aliases above. +- The command-injection scan outcome gains an additive `normalization` object + (`droppedFindings`, `duplicateFindings`, `issues`). Existing outcome fields are unchanged. +- Other scan jobs (deserialization, redirect) are unchanged. Adopt normalization by passing the + analyzer result through `normalizeAnalysisResult` before persisting. diff --git a/docs/FINDING_LOCATION_MAPPING.md b/docs/FINDING_LOCATION_MAPPING.md new file mode 100644 index 00000000..0b38de5c --- /dev/null +++ b/docs/FINDING_LOCATION_MAPPING.md @@ -0,0 +1,109 @@ +# Stable Finding Location Mapping + +A finding's line number goes stale as soon as code above it changes. Location mapping stores a +content **anchor** with each finding, so the finding can be re-pointed at the same code in a +newer revision of the file, or reported as lost when the code no longer exists. + +| Layer | Location | +| --- | --- | +| Anchor + matcher (pure, engine) | `libs/engine/core/location-mapper.ts` | +| Relocation planning (pure, API) | `apps/api/src/findings/finding-location.ts` | +| Persistence | `FindingsService.relocateFindings` (`apps/api/src/findings/findings.service.ts`) | +| Anchor capture at scan time | `apps/api/src/scanner/command-injection-scan.job.ts` | +| Fixtures | `tests/fixtures/location-mapping/` | +| Tests | `libs/engine/core/location-mapper.spec.ts`, `apps/api/src/findings/__tests__/findings.relocation.spec.ts` | + +## How it works + +`createLocationAnchor(content, { startLine, endLine })` records: + +- the hash of the flagged first line and of the whole flagged span, +- hashes of up to 3 nearest non-blank lines above and below (nearest first). + +Lines are hashed after trimming and collapsing whitespace. Line endings, indentation and blank-line +changes therefore never affect a match. + +`mapLocation(anchor, content)` looks for lines with the same hash and scores each candidate: + +``` +confidence = 0.2 (line text matches) + + 0.3 (whole span matches) + + 0.5 * context agreement +``` + +Context agreement is the average, over the lines above and below, of how much of the nearest-first +neighbour run still matches. It stops at the first mismatch. The best candidate must reach +`minConfidence` (default 0.65). Ties are broken by distance from the old line, then by the lowest line. + +| Status | Meaning | +| --- | --- | +| `exact` | Same lines, same place. | +| `moved` | Same code found at a different line. | +| `modified` | The flagged line was edited, but exactly one place has identical surroundings on both sides (confidence 0.65). | +| `ambiguous` | Several equally good matches (identical copies). The closest was chosen. | +| `lost` | No acceptable match. The last known line and anchor are kept. | + +An edited line is never pinned to an identical-looking line elsewhere: the surrounding context has to +agree as well. A line that moved to a place with entirely different surroundings is reported `lost` +rather than guessed. Callers can lower `minConfidence` to accept weaker matches. + +## Public interface + +```ts +import { createLocationAnchor, mapLocation } from '@engine/core/location-mapper'; + +const anchor = createLocationAnchor(oldContent, { startLine: 9, endLine: 9 }); +const mapping = mapLocation(anchor, newContent); +// { status: 'moved', startLine: 17, endLine: 17, confidence: 1, delta: 8 } +``` + +API findings gain two optional fields: `locationAnchor` and `locationStatus`. +`CreateFindingInput.locationAnchor` stores an anchor at creation. + +```ts +findingsService.relocateFindings({ + organizationId, + repositoryId, + files: { 'src/runner.ts': newContent }, // or Map + deletedFiles: ['src/old.ts'], // optional + options: { minConfidence: 0.65 }, // optional +}); +// -> { examined, unchanged, relocated, lost, skippedNoAnchor, skippedFileNotProvided } +``` + +## Determinism + +The same anchor, content and options always produce the same mapping. Nothing depends on the clock, +input order or shared state. Relocation is idempotent: running it again over the same contents +reports every finding as `unchanged`. `locationStatus` records the outcome of the last relocation that +changed something (`moved`, `modified`, `ambiguous`, `lost`), and is kept on reruns that find the code +exactly where it was left. After a successful move the anchor is re-captured at the new +location, so mappings chain across many revisions. + +## Failure semantics and configuration + +- **Tenant isolation**: `relocateFindings` only reads and writes findings of the given + `organizationId` and `repositoryId`. Both are required (`VALIDATION_ERROR`, 400). +- **Unsupported input**: findings without a usable anchor or `filePath` are skipped, not failed + (`skippedNoAnchor`). Corrupt anchors are treated the same way. Paths are normalized with + `normalizeFindingPath`, and unsafe paths (`..`, absolute) are ignored. +- **Partial failure**: only files present in `files` are processed. A file missing from the snapshot + is `skippedFileNotProvided` and never marks findings lost. Use `deletedFiles` to say a file is gone. +- **Lost findings**: the line and anchor stay as they were, so a later revision where the code + returns recovers the finding. +- **Retries**: safe, because the operation is idempotent. +- **Never throws** on malformed content or anchors. `mapLocation` returns `lost`, and + `createLocationAnchor` returns `undefined` when the span is outside the file. +- **Cost**: `contextLines` (default 3) and `maxCandidates` (default 500) bound the work per lookup. + Build a `FileLineIndex` once per file to map many anchors. + +## Backward compatibility + +- Findings created before this feature have no anchor and are left untouched. +- The fingerprint ignores line numbers, so relocating a finding never changes its identity or + its history. +- `Finding.line` keeps its meaning (1-based start line); relocation only updates it. +- The anchor has a `version` (currently `1`). An unknown version is ignored like a missing anchor. +- The command-injection scan job now attaches an anchor to each finding it creates. Other jobs are + unchanged and can pass `locationAnchor` the same way. +- **Not covered**: file renames. A renamed file is seen as a deleted file plus a new one. diff --git a/docs/JOB_PRIORITY_QUEUES.md b/docs/JOB_PRIORITY_QUEUES.md new file mode 100644 index 00000000..5ecf08bb --- /dev/null +++ b/docs/JOB_PRIORITY_QUEUES.md @@ -0,0 +1,88 @@ +# Job Priority Queues + +Scans are not equally urgent. A blocking CI check should not wait behind a nightly full-repository +audit. `JobPriorityQueue` orders queued work by priority and keeps low-priority work from starving. + +| Piece | Location | +| --- | --- | +| Queue, priority resolution, BullMQ mapping | `apps/api/src/queue/priority-queue.ts` | +| Unit tests (normal, boundary, failure) | `apps/api/src/queue/priority-queue.spec.ts` | + +The module is self-contained: no I/O, no clock reads, no dependency on other queue modules. Callers +pass `now`, as with the other queue primitives, so behaviour is deterministic. + +## Priorities + +| Priority | BullMQ value | Intended use | +| --- | --- | --- | +| `critical` | 1 | Internal, blocking work. Reserved: not grantable to ordinary callers. | +| `high` | 2 | Interactive and CI scans. The default ceiling for requests. | +| `normal` | 3 | Default. | +| `low` | 4 | Scheduled and bulk scans. | + +Within a priority, jobs run in arrival order. + +## Interface + +```ts +import { JobPriorityQueue, resolveJobPriority } from './queue/priority-queue'; + +const queue = new JobPriorityQueue({ maxSizePerPriority: 10_000, agingMs: 30_000 }); + +// Untrusted input is clamped to a ceiling ('high' by default) and validated. +const priority = resolveJobPriority(request.priority); + +const result = queue.enqueue(scanId, request, { priority }, now); +if (!result.accepted) { + // result.reason is 'duplicate' or 'queue_full'. Apply backpressure. +} + +const job = queue.dequeue(now); // QueuedJob | undefined +queue.remove(scanId); // cancel +queue.oldestEnqueuedAt(); // feed QueueAgeSlo.evaluate +queue.sizeByPriority(); // metrics +queue.stats(); // counters +``` + +Using it with BullMQ: `{ ...withRetryOptions(), priority: toBullMqPriority(priority) }`. + +## Starvation protection (aging) + +Every `agingMs` a job has waited promotes it one level, up to `critical`. A promoted job competes with +other jobs at that level by arrival order, so a `low` job waits at most about `3 x agingMs` before +it is ahead of any newer job, even under a constant stream of `critical` work. `effectivePriority` +on the dequeued job shows the level it was competing at, and `stats().promoted` counts jobs that left +ahead of their original level. + +## Configuration and secure defaults + +| Option | Default | Notes | +| --- | --- | --- | +| `maxSizePerPriority` | 10,000 | Capacity of each level. Per-level limits mean a flood of low-priority work cannot block a `critical` job from being accepted. | +| `agingMs` | 30,000 | Must be a positive integer. Use a very large value to effectively disable aging. | + +- **Callers cannot escalate themselves.** `resolveJobPriority(requested, ceiling = 'high')` lowers + anything above the ceiling. Pass `'critical'` as the ceiling only for trusted callers. +- **Unrecognised priorities are errors**, never silently promoted or demoted. +- Job ids are non-empty strings of at most 256 characters. + +## Failure semantics + +- **Cannot accept**: `enqueue` returns `{ accepted: false, reason: 'duplicate' | 'queue_full' }` rather + than throwing, so callers can retry later or return a 429-style response. A duplicate is an id that + is still queued; once a job is dequeued or removed its id may be enqueued again. +- **Invalid arguments** (bad id, unknown priority, non-finite `now`, invalid config) throw + `TypeError` or `RangeError`, and the queue is left unchanged. +- **Clock skew**: a `now` earlier than a job's enqueue time counts as zero wait, never a negative one. +- **Cancelled jobs** are dropped lazily and the backing arrays are compacted, so memory stays bounded + under enqueue/cancel churn. + +## Dependencies and limits + +- Reads nothing from other components. It complements `RepositoryScanQuota` (admission per repository), + `ScanDeduplicator` and `QueueAgeSlo`, which a dispatcher can combine with it. +- It is an in-process queue. It does not persist jobs, and it does not make Redis-backed BullMQ queues + priority-aware by itself; use `toBullMqPriority` for that. A retried job re-enters as a new job and + starts aging again. +- Fairness between tenants at the same priority is not part of this module. Use the per-repository + quota for that. diff --git a/docs/RELIABILITY_LOAD_TESTING.md b/docs/RELIABILITY_LOAD_TESTING.md new file mode 100644 index 00000000..41843d2a --- /dev/null +++ b/docs/RELIABILITY_LOAD_TESTING.md @@ -0,0 +1,103 @@ +# Reliability Load Testing + +A deterministic load suite for the scan pipeline. It drives the production queue primitives under +heavy, bursty and faulty load and fails when a reliability invariant breaks or an SLO target is missed. + +| Piece | Location | +| --- | --- | +| Simulator (wires the real components) | `tests/load/scan-pipeline.simulator.ts` | +| Scenarios | `tests/load/scenarios.ts` | +| SLO targets and evaluation | `tests/load/slo.ts` | +| Seeded RNG, event scheduler, percentile stats | `tests/load/harness/` | +| Unit tests (harness, config validation) | `tests/load/harness/harness.spec.ts` | +| Scenario tests | `tests/load/scan-pipeline.load.spec.ts` | +| Integration test (real job handler) | `tests/load/scan-handler.load.spec.ts` | +| CLI report | `scripts/run-load-test.ts` | + +## How it works + +The simulator runs a pool of workers in **virtual time**: nothing sleeps, so ten simulated minutes +run in well under a second and give identical results on every machine. Only the clock, the scan +bodies and the faults are simulated. Every admission, lease and checkpoint decision is made by the +real code in `apps/api/src/queue/`: + +| Concern | Component | +| --- | --- | +| Duplicate submissions | `ScanDeduplicator` | +| Per-repository concurrency and rate | `RepositoryScanQuota` | +| Exclusive ownership, fencing tokens | `ScanLeaseManager` | +| Resuming interrupted scans | `ScanCheckpointStore` | +| Queue-age health | `QueueAgeSlo` | + +Dependencies: the queue modules above (read-only, nothing is modified) and, for the integration +test, `CommandInjectionScanJobHandler` and `FindingsService`. No Redis, BullMQ, network or database +is used. + +## Running + +```bash +npm run test:load # all load specs +LOAD_SCALE=20 npm run test:load # 20x longer runs (1-100) +npm run load:report # every scenario +npm run load:report -- --scenario worker-crash --seed 7 --scale 5 +``` + +`load:report` exits 1 on any invariant violation or missed SLO, so it can gate CI. A failing run is +reproducible from its scenario, seed and scale alone. + +## Scenarios + +| Scenario | What it stresses | +| --- | --- | +| `steady` | 12 tenants at a comfortable load: baseline latency, no faults. | +| `burst` | 150 scans from one repository in 2 s against a tight window quota. | +| `noisy-neighbor` | One repository submits about 15x the others. Others must stay responsive. | +| `worker-crash` | Workers die mid-scan, or freeze past their lease and wake up as zombies. | +| `duplicate-storm` | Clients re-send the same revision up to 4 times. | +| `flaky-poison` | Random failures with backoff, plus a repository whose scans always fail. | +| `chaos` | All of the above together; the tests run it over five seeds. | + +## Invariants + +These are checked continuously and reported in `LoadReport.violations`. Any entry fails the run. + +- A scan commits at most once, and every unique scan ends completed or dead-lettered (none lost). +- A stale fencing token can never renew a lease, and a frozen worker that wakes up is refused. +- A worker never loses a lease it is actively renewing. +- The quota's in-flight count always matches the scans actually running, and never exceeds the + per-repository limit. No window ever admits more than its allowance. +- Checkpoints never move backwards, and an interruption replays at most `checkpointEvery` items per + crash, stall or failure. +- Duplicate revisions never produce a second scan. +- After draining there are no live leases, no leaked quota slots and no queued work. + +## SLO targets + +`tests/load/slo.ts` sets, per scenario, a ceiling for the p95 wait of non-noisy tenants, a ceiling +for the dead-letter ratio, and (for `steady`) a minimum queue-age compliance. Targets are generous +on purpose so they catch real regressions, such as a quota change that starves quiet tenants. +Tighten them as the system improves. + +## Failure semantics and secure defaults + +- **Invalid configuration** (`workers < 1`, lease TTL shorter than two item durations, fault rates + above 1, duplicate tenants, and so on) throws a `RangeError` before any work runs. +- **Broken invariants never throw.** The run continues and lists every violation (capped at 50) so + one run shows everything wrong. +- **Bounded by design:** `LOAD_SCALE` and `--scale` are limited to 100, and a run stops with a + violation after 5,000,000 events instead of hanging. +- **Safe to run anywhere:** no network, no filesystem writes, no wall-clock timers, and + `Math.random` is never used. The integration test uses only tenant IDs it generates. + +## Adding a scenario + +1. Add a name to `SCENARIO_NAMES` and a `case` in `buildScenario` (`scenarios.ts`). +2. Add its targets to `SLO_TARGETS` (`slo.ts`). The compiler will complain if you forget. +3. Add a `describe` block to `scan-pipeline.load.spec.ts` asserting what makes the scenario + distinctive. The generic invariant and determinism tests pick it up automatically. + +## Limits + +The suite exercises the logic of the queue primitives, not the throughput of Redis or BullMQ. It +runs in one process on one logical clock, so it cannot show network partitions between workers and +a shared store or real-clock behaviour. Pair it with an environment-level soak test for those. diff --git a/libs/engine/core/index.ts b/libs/engine/core/index.ts index 7f691849..c2404b2c 100644 --- a/libs/engine/core/index.ts +++ b/libs/engine/core/index.ts @@ -27,5 +27,11 @@ export * from "./rule-lifecycle"; // Analyzer compatibility matrix (#1025) export * from "./compatibility-matrix"; +// Analyzer result normalization +export * from "./result-normalizer"; + +// Stable finding location mapping +export * from "./location-mapper"; + // Analyzer plugin contract and lifecycle (#1154) export * from "./analyzer-plugin-contract"; diff --git a/libs/engine/core/location-mapper.spec.ts b/libs/engine/core/location-mapper.spec.ts new file mode 100644 index 00000000..e9abfa89 --- /dev/null +++ b/libs/engine/core/location-mapper.spec.ts @@ -0,0 +1,162 @@ +import * as fs from "fs"; +import * as path from "path"; +import { + buildFileLineIndex, + createLocationAnchor, + isLocationAnchor, + mapLocation, +} from "./location-mapper"; + +const FIXTURES = path.resolve(__dirname, "../../../tests/fixtures/location-mapping"); +const read = (name: string) => fs.readFileSync(path.join(FIXTURES, name), "utf8"); + +const v1 = read("runner.v1.ts"); +const v2 = read("runner.v2.ts"); +const v3 = read("runner.v3.ts"); +const v4 = read("runner.v4-rewritten.ts"); + +/** 1-based line number of the nth (0-based) line whose trimmed text equals `text`. */ +function lineOf(content: string, text: string, nth = 0): number { + const hits = content + .split("\n") + .map((l, i) => ({ l: l.trim(), n: i + 1 })) + .filter((x) => x.l === text); + return hits[nth].n; +} + +describe("createLocationAnchor", () => { + it("captures hashes and non-blank context, ignoring blank lines", () => { + const a = createLocationAnchor(v1, { startLine: 9 })!; + expect(isLocationAnchor(a)).toBe(true); + expect(a.startLine).toBe(9); + expect(a.endLine).toBe(9); + expect(a.before).toHaveLength(3); + expect(a.after).toHaveLength(3); + }); + + it("returns undefined for lines outside the file or invalid spans", () => { + expect(createLocationAnchor(v1, { startLine: 999 })).toBeUndefined(); + expect(createLocationAnchor(v1, { startLine: 0 })).toBeUndefined(); + expect(createLocationAnchor(v1, { startLine: Number.NaN })).toBeUndefined(); + expect(createLocationAnchor(undefined as unknown as string, { startLine: 1 })).toBeUndefined(); + }); + + it("clamps an overlong or inverted span", () => { + expect(createLocationAnchor(v1, { startLine: 9, endLine: 2 })!.endLine).toBe(9); + expect(createLocationAnchor(v1, { startLine: 9, endLine: 9999 })!.endLine).toBe(15); + }); + + it("is deterministic and rejects malformed anchors", () => { + expect(createLocationAnchor(v1, { startLine: 9 })).toEqual(createLocationAnchor(v1, { startLine: 9 })); + expect(isLocationAnchor({ version: 99 })).toBe(false); + expect(isLocationAnchor(null)).toBe(false); + }); +}); + +describe("mapLocation — realistic repository revisions", () => { + const lookupAnchor = createLocationAnchor(v1, { startLine: 9 })!; + const tailAnchor = createLocationAnchor(v1, { startLine: 14 })!; + + it("reports exact for an unchanged file", () => { + const m = mapLocation(lookupAnchor, v1); + expect(m).toMatchObject({ status: "exact", startLine: 9, delta: 0, confidence: 1 }); + }); + + it("follows each finding through inserted code, not to the look-alike line", () => { + const lookup = mapLocation(lookupAnchor, v2); + const tail = mapLocation(tailAnchor, v2); + expect(lookup.status).toBe("moved"); + expect(lookup.startLine).toBe(lineOf(v2, "return exec(cmd);", 0)); + expect(tail.status).toBe("moved"); + expect(tail.startLine).toBe(lineOf(v2, "return exec(cmd);", 1)); + expect(lookup.delta).toBe(lookup.startLine - 9); + }); + + it("finds an edited line from its unchanged surroundings and never jumps to the identical line", () => { + const m = mapLocation(lookupAnchor, v3); + expect(m.status).toBe("modified"); + expect(m.startLine).toBe(9); + // the unedited sibling still maps exactly + expect(mapLocation(tailAnchor, v3)).toMatchObject({ status: "exact", startLine: 14 }); + }); + + it("reports lost, keeping the last known location, when the code is gone", () => { + expect(mapLocation(lookupAnchor, v4)).toEqual({ + status: "lost", + startLine: 9, + endLine: 9, + confidence: 0, + delta: 0, + }); + }); + + it("is stable across line endings and indentation changes", () => { + const crlf = v2.replace(/\n/g, "\r\n"); + const reindented = v2.replace(/^ {2}/gm, "\t"); + const expected = mapLocation(lookupAnchor, v2); + expect(mapLocation(lookupAnchor, crlf)).toEqual(expected); + expect(mapLocation(lookupAnchor, reindented)).toEqual(expected); + }); + + it("is chainable: re-anchoring after each move keeps the mapping exact", () => { + const m2 = mapLocation(lookupAnchor, v2); + const reanchored = createLocationAnchor(v2, { startLine: m2.startLine })!; + expect(mapLocation(reanchored, v2)).toMatchObject({ status: "exact", startLine: m2.startLine }); + }); + + it("gives identical results from a string and a prebuilt index", () => { + expect(mapLocation(lookupAnchor, buildFileLineIndex(v2))).toEqual(mapLocation(lookupAnchor, v2)); + }); +}); + +describe("mapLocation — multi-line spans", () => { + it("preserves span length when the block moves", () => { + const anchor = createLocationAnchor(v1, { startLine: 8, endLine: 9 })!; + const m = mapLocation(anchor, v2); + expect(m.status).toBe("moved"); + expect(m.endLine - m.startLine).toBe(1); + expect(m.startLine).toBe(lineOf(v2, "const cmd = `nslookup ${name}`;")); + }); +}); + +describe("mapLocation — ambiguity and failure semantics", () => { + it("flags ambiguous when identical blocks tie, choosing the closest then lowest line", () => { + const anchor = createLocationAnchor("a\nX\nb", { startLine: 2 })!; + const m = mapLocation(anchor, "c\na\nX\nb\nd\na\nX\nb"); + expect(m.status).toBe("ambiguous"); + expect(m.startLine).toBe(3); + }); + + it("prefers the original line when copies tie", () => { + const anchor = createLocationAnchor("a\nX\nb", { startLine: 2 })!; + expect(mapLocation(anchor, "a\nX\nb\n\na\nX\nb").status).toBe("exact"); + }); + + it("does not guess when only the line text survives with different context", () => { + const anchor = createLocationAnchor("a\nb\nTARGET\nc\nd", { startLine: 3 })!; + expect(mapLocation(anchor, "w\nx\nTARGET\ny\nz\n").status).toBe("lost"); + }); + + it("lets callers loosen the threshold", () => { + const anchor = createLocationAnchor("a\nb\nTARGET\nc\nd", { startLine: 3 })!; + expect(mapLocation(anchor, "w\nx\nTARGET\ny\nz\n", { minConfidence: 0.5 }).status).toBe("exact"); + }); + + it("treats blank anchors as trustworthy only at their original position", () => { + const anchor = createLocationAnchor("a\n\nb", { startLine: 2 })!; + expect(mapLocation(anchor, "a\n\nb").status).toBe("exact"); + expect(mapLocation(anchor, "x\na\n\nb").status).toBe("lost"); + }); + + it("never throws on bad input", () => { + const anchor = createLocationAnchor(v1, { startLine: 9 })!; + expect(mapLocation(anchor, "").status).toBe("lost"); + expect(mapLocation(anchor, undefined as unknown as string).status).toBe("lost"); + expect(mapLocation({} as never, v1).status).toBe("lost"); + }); + + it("is deterministic for repeated calls", () => { + const anchor = createLocationAnchor(v1, { startLine: 9 })!; + expect(mapLocation(anchor, v2)).toEqual(mapLocation(anchor, v2)); + }); +}); diff --git a/libs/engine/core/location-mapper.ts b/libs/engine/core/location-mapper.ts new file mode 100644 index 00000000..3de9ae6c --- /dev/null +++ b/libs/engine/core/location-mapper.ts @@ -0,0 +1,307 @@ +import { createHash } from "crypto"; + +/** + * Stable finding location mapping. + * + * A finding's line number goes stale as soon as code above it changes. Instead + * of trusting the line, we store a content *anchor*: hashes of the flagged + * lines plus a few neighbouring non-blank lines. Given a later revision of the + * file, `mapLocation` finds where that code moved to, or says it can't. + * + * Guarantees: + * - Pure, no I/O, no clock, tenant-agnostic. Never throws on bad input. + * - Deterministic: same anchor + same content + same options always give the + * same answer, including on ties (closest to the old line, then lowest line). + * - Honest: weak matches are reported as `ambiguous` or `lost`, never guessed + * at with high confidence. + */ + +export const LOCATION_ANCHOR_VERSION = 1; +export const DEFAULT_CONTEXT_LINES = 3; +export const DEFAULT_MIN_CONFIDENCE = 0.65; +export const DEFAULT_MAX_CANDIDATES = 500; +/** Confidence reported when only the surroundings of an edited line matched. */ +const MODIFIED_CONFIDENCE = 0.65; +/** How far past blank lines we look for a non-blank neighbour. */ +const MAX_BLANK_SCAN = 50; +/** Hard limit on anchored span so a bad finding can't make us hash a whole file. */ +export const MAX_ANCHOR_SPAN_LINES = 200; + +export interface SourceSpan { + /** 1-based inclusive. */ + startLine: number; + endLine?: number; +} + +export interface LocationAnchor { + version: number; + /** Line numbers where the anchor was taken (the "last known" location). */ + startLine: number; + endLine: number; + /** Hash of the normalized first flagged line. Empty when that line is blank. */ + lineHash: string; + /** Hash of all normalized lines in the span. */ + spanHash: string; + /** Hashes of the nearest non-blank lines above, nearest first. */ + before: string[]; + /** Hashes of the nearest non-blank lines below, nearest first. */ + after: string[]; +} + +export type LocationStatus = + /** Same lines, same place. */ + | "exact" + /** Same code, found at a different line. */ + | "moved" + /** The flagged line was edited, but its surroundings match exactly. */ + | "modified" + /** Several equally good matches; the closest one was chosen. */ + | "ambiguous" + /** No acceptable match. Line numbers are the last known ones. */ + | "lost"; + +export interface LocationMapping { + status: LocationStatus; + startLine: number; + endLine: number; + /** 0..1. 0 when lost. */ + confidence: number; + /** startLine - anchor.startLine. */ + delta: number; +} + +export interface LocationMapOptions { + /** Neighbouring non-blank lines hashed on each side. Default 3. */ + contextLines?: number; + /** Minimum confidence to accept a match. Default 0.65. */ + minConfidence?: number; + /** Max candidate positions scored per lookup. Default 500. */ + maxCandidates?: number; +} + +/** Pre-split, pre-hashed file; build once and reuse for many anchors. */ +export interface FileLineIndex { + lineCount: number; + hashes: string[]; + positions: Map; +} + +function hashLine(text: string): string { + const norm = text.trim().replace(/\s+/g, " "); + if (norm === "") return ""; + return createHash("sha256").update(norm).digest("hex").slice(0, 12); +} + +function hashSpan(hashes: string[]): string { + return createHash("sha256").update(hashes.join(",")).digest("hex").slice(0, 16); +} + +function splitLines(content: string): string[] { + const lines = content.split(/\r\n|\r|\n/); + // A trailing newline does not start a new line. + if (lines.length > 1 && lines[lines.length - 1] === "") lines.pop(); + return lines; +} + +export function buildFileLineIndex(content: string): FileLineIndex { + const hashes = splitLines(typeof content === "string" ? content : "").map(hashLine); + const positions = new Map(); + hashes.forEach((h, i) => { + if (h === "") return; + const list = positions.get(h); + if (list) list.push(i); + else positions.set(h, [i]); + }); + return { lineCount: hashes.length, hashes, positions }; +} + +function neighbours(hashes: string[], from: number, step: 1 | -1, count: number): string[] { + const out: string[] = []; + let scanned = 0; + for (let i = from + step; i >= 0 && i < hashes.length && out.length < count; i += step) { + if (hashes[i] === "") { + if (++scanned > MAX_BLANK_SCAN) break; + continue; + } + out.push(hashes[i]); + } + return out; +} + +function toInt(n: unknown): number | undefined { + return typeof n === "number" && Number.isFinite(n) && n >= 1 ? Math.floor(n) : undefined; +} + +/** + * Captures an anchor for `span` in `content`. Returns undefined when the span + * doesn't lie inside the file (so callers store no anchor rather than a bogus one). + */ +export function createLocationAnchor( + content: string, + span: SourceSpan, + options: LocationMapOptions = {}, +): LocationAnchor | undefined { + const startLine = toInt(span?.startLine); + if (startLine === undefined || typeof content !== "string") return undefined; + let endLine = toInt(span.endLine) ?? startLine; + if (endLine < startLine) endLine = startLine; + + const index = buildFileLineIndex(content); + if (startLine > index.lineCount) return undefined; + endLine = Math.min(endLine, index.lineCount, startLine + MAX_ANCHOR_SPAN_LINES - 1); + + const n = options.contextLines ?? DEFAULT_CONTEXT_LINES; + const s = startLine - 1; + const e = endLine - 1; + return { + version: LOCATION_ANCHOR_VERSION, + startLine, + endLine, + lineHash: index.hashes[s], + spanHash: hashSpan(index.hashes.slice(s, e + 1)), + before: neighbours(index.hashes, s, -1, n), + after: neighbours(index.hashes, e, 1, n), + }; +} + +export function isLocationAnchor(value: unknown): value is LocationAnchor { + const a = value as LocationAnchor | null; + return ( + a !== null && + typeof a === "object" && + a.version === LOCATION_ANCHOR_VERSION && + toInt(a.startLine) !== undefined && + toInt(a.endLine) !== undefined && + a.endLine >= a.startLine && + typeof a.lineHash === "string" && + typeof a.spanHash === "string" && + Array.isArray(a.before) && + Array.isArray(a.after) && + a.before.every((h) => typeof h === "string") && + a.after.every((h) => typeof h === "string") + ); +} + +function sideScore(expected: string[], actual: string[]): number { + if (expected.length === 0) return actual.length === 0 ? 1 : 0.5; + // Only the unbroken run starting at the flagged line counts: a mismatch + // right next to the code means the neighbourhood is not the same place. + let run = 0; + while (run < expected.length && expected[run] === actual[run]) run++; + return run / expected.length; +} + +function lost(anchor: LocationAnchor): LocationMapping { + return { status: "lost", startLine: anchor.startLine, endLine: anchor.endLine, confidence: 0, delta: 0 }; +} + +interface Candidate { + line: number; // 1-based + confidence: number; + spanMatches: boolean; +} + +/** + * Finds where `anchor` lives in `target` (file content or a prebuilt index). + * + * Confidence = 0.2 (line text matches) + 0.3 (whole span matches) + + * 0.5 * context agreement (average, over the lines above and below, of how + * much of the nearest-first neighbour run still matches). Context must + * therefore agree at least ~30% for a match to be accepted at the default + * threshold, so an edited line is never silently pinned to an identical-looking + * line elsewhere in the file. + */ +export function mapLocation( + anchor: LocationAnchor, + target: string | FileLineIndex, + options: LocationMapOptions = {}, +): LocationMapping { + if (!isLocationAnchor(anchor)) { + return { status: "lost", startLine: 1, endLine: 1, confidence: 0, delta: 0 }; + } + const index = typeof target === "string" ? buildFileLineIndex(target) : target; + const minConfidence = options.minConfidence ?? DEFAULT_MIN_CONFIDENCE; + const maxCandidates = options.maxCandidates ?? DEFAULT_MAX_CANDIDATES; + const n = Math.max(anchor.before.length, anchor.after.length, options.contextLines ?? 0); + const span = anchor.endLine - anchor.startLine; + + const clampSpan = (start: number) => ({ + startLine: start, + endLine: Math.min(start + span, Math.max(index.lineCount, start)), + }); + + const score = (i: number): Candidate => { + const spanHashes = index.hashes.slice(i, i + span + 1); + const spanMatches = spanHashes.length === span + 1 && hashSpan(spanHashes) === anchor.spanHash; + const before = sideScore(anchor.before, neighbours(index.hashes, i, -1, n)); + const after = sideScore(anchor.after, neighbours(index.hashes, i + span, 1, n)); + return { + line: i + 1, + spanMatches, + confidence: 0.2 + (spanMatches ? 0.3 : 0) + 0.25 * before + 0.25 * after, + }; + }; + + const better = (a: Candidate, b: Candidate): number => + b.confidence - a.confidence || + Math.abs(a.line - anchor.startLine) - Math.abs(b.line - anchor.startLine) || + a.line - b.line; + + // Blank anchor line: only the original position can be trusted. + if (anchor.lineHash === "") { + const same = index.hashes.slice(anchor.startLine - 1, anchor.endLine); + if (same.length === span + 1 && hashSpan(same) === anchor.spanHash) { + return { status: "exact", ...clampSpan(anchor.startLine), confidence: 1, delta: 0 }; + } + return lost(anchor); + } + + const positions = index.positions.get(anchor.lineHash) ?? []; + if (positions.length > 0) { + const candidates = positions.slice(0, maxCandidates).map(score).sort(better); + const best = candidates[0]; + if (best.confidence >= minConfidence) { + const runnerUp = candidates[1]; + const sameLine = best.line === anchor.startLine; + let status: LocationStatus; + if (sameLine && best.spanMatches) status = "exact"; + else if ( + runnerUp && + runnerUp.confidence === best.confidence && + runnerUp.line !== best.line + ) status = "ambiguous"; + else status = sameLine ? "modified" : "moved"; + return { + status, + ...clampSpan(best.line), + confidence: Number(best.confidence.toFixed(3)), + delta: best.line - anchor.startLine, + }; + } + } + + // The flagged line itself changed. Accept only if both neighbourhoods agree + // perfectly, there is something to compare, and exactly one place matches. + if (anchor.before.length > 0 && anchor.after.length > 0) { + const matches: number[] = []; + const first = anchor.before[0]; + for (let i = 0; i < index.lineCount && matches.length < 2; i++) { + if (index.hashes[i] === "" || index.hashes[i] === anchor.lineHash) continue; + // `i` is the candidate start line; its nearest non-blank neighbours must match. + const b = neighbours(index.hashes, i, -1, anchor.before.length); + if (b[0] !== first) continue; + const a = neighbours(index.hashes, i + span, 1, anchor.after.length); + if (sideScore(anchor.before, b) === 1 && sideScore(anchor.after, a) === 1) matches.push(i + 1); + } + if (matches.length === 1 && MODIFIED_CONFIDENCE >= minConfidence) { + return { + status: "modified", + ...clampSpan(matches[0]), + confidence: MODIFIED_CONFIDENCE, + delta: matches[0] - anchor.startLine, + }; + } + } + + return lost(anchor); +} diff --git a/libs/engine/core/result-normalizer.spec.ts b/libs/engine/core/result-normalizer.spec.ts new file mode 100644 index 00000000..b47fea25 --- /dev/null +++ b/libs/engine/core/result-normalizer.spec.ts @@ -0,0 +1,182 @@ +import * as fs from "fs"; +import * as path from "path"; +import { Severity } from "./analyzer-interface"; +import { + NORMALIZATION_SCHEMA_VERSION, + normalizeAnalysisResult, + normalizeFindingPath, + normalizeFindings, + normalizeSeverity, +} from "./result-normalizer"; + +const FIXTURES = path.resolve(__dirname, "../../../tests/fixtures/result-normalization"); +const load = (name: string) => JSON.parse(fs.readFileSync(path.join(FIXTURES, name), "utf8")); + +describe("normalizeFindingPath", () => { + it("canonicalizes separators, dot segments and parent segments", () => { + expect(normalizeFindingPath("src\\a\\b.ts")).toBe("src/a/b.ts"); + expect(normalizeFindingPath("./src//a/./b/../c.ts")).toBe("src/a/c.ts"); + }); + + it("rejects traversal, empty, control-char and non-string input", () => { + expect(normalizeFindingPath("../x.ts")).toBeUndefined(); + expect(normalizeFindingPath("a/../../x.ts")).toBeUndefined(); + expect(normalizeFindingPath(" ")).toBeUndefined(); + expect(normalizeFindingPath("a\u0000b")).toBeUndefined(); + expect(normalizeFindingPath(42)).toBeUndefined(); + }); + + it("rejects absolute paths unless inside rootDir", () => { + expect(normalizeFindingPath("/work/app/a.ts")).toBeUndefined(); + expect(normalizeFindingPath("/work/app/a.ts", "/work/app")).toBe("a.ts"); + expect(normalizeFindingPath("/work/other/a.ts", "/work/app")).toBeUndefined(); + expect(normalizeFindingPath("/work/application/a.ts", "/work/app")).toBeUndefined(); + }); +}); + +describe("normalizeSeverity", () => { + it("maps aliases case-insensitively and rejects unknown values", () => { + expect(normalizeSeverity("ERROR")).toBe(Severity.HIGH); + expect(normalizeSeverity(" Warning ")).toBe(Severity.MEDIUM); + expect(normalizeSeverity("note")).toBe(Severity.INFO); + expect(normalizeSeverity("bogus")).toBeUndefined(); + expect(normalizeSeverity(3)).toBeUndefined(); + }); +}); + +describe("normalizeAnalysisResult — realistic repository output", () => { + const raw = load("realistic-mixed-output.json"); + const result = normalizeAnalysisResult(raw, { analyzerName: "mixed" }); + + it("produces canonical paths, severities and line ranges", () => { + expect(result.schemaVersion).toBe(NORMALIZATION_SCHEMA_VERSION); + expect(result.analyzerName).toBe("mixed"); + const files = result.findings.map((f) => f.location.file); + expect(files).toEqual([ + "contracts/Vault.sol", + "scripts/deploy.py", + "src/services/runner.ts", + "src/services/runner.ts", + ]); + const tpl = result.findings.find((f) => f.ruleId === "cmd-004")!; + expect(tpl.severity).toBe(Severity.HIGH); + expect(tpl.location.startLine).toBe(57); + expect(tpl.location.endLine).toBe(57); + expect(tpl.message).toBe("Command built with template literal\nuse execFile with an argument array"); + expect(result.findings.find((f) => f.ruleId === "cmd-002")!.location.startLine).toBe(9); + }); + + it("collapses duplicates keeping the highest severity, and recomputes the summary", () => { + const cmd001 = result.findings.filter((f) => f.ruleId === "cmd-001"); + expect(cmd001).toHaveLength(1); + expect(cmd001[0].severity).toBe(Severity.CRITICAL); + expect(result.counts.duplicates).toBe(1); + expect(result.summary).toEqual({ critical: 1, high: 1, medium: 2, low: 0, info: 0 }); + expect(result.totalEstimatedGasSavings).toBe(2100); + }); + + it("keeps run-dependent data out of the deterministic body", () => { + expect(result.telemetry.analysisTimeMs).toBe(183); + expect(result).not.toHaveProperty("analysisTime"); + expect(result.errors).toEqual([{ file: "docs/broken.md", message: "unsupported encoding" }]); + }); + + it("is deterministic regardless of input order and timing", () => { + const shuffled = { ...raw, analysisTime: 9999, findings: [...raw.findings].reverse() }; + const again = normalizeAnalysisResult(shuffled, { analyzerName: "mixed" }); + expect(JSON.stringify({ ...again, telemetry: undefined })).toBe( + JSON.stringify({ ...result, telemetry: undefined }), + ); + expect(again.findings.map((f) => f.key)).toEqual(result.findings.map((f) => f.key)); + }); + + it("sorts metadata keys so serialization is stable", () => { + const vault = result.findings.find((f) => f.ruleId === "gas-007")!; + expect(JSON.stringify(vault.metadata)).toBe('{"a":{"b":3,"y":2},"z":1}'); + }); + + it("is idempotent", () => { + const twice = normalizeAnalysisResult(result, { analyzerName: "mixed" }); + expect(twice.findings).toEqual(result.findings); + expect(twice.summary).toEqual(result.summary); + }); +}); + +describe("normalizeAnalysisResult — malformed input regression", () => { + const raw = load("regression-malformed-findings.json"); + const result = normalizeAnalysisResult(raw); + + it("never throws and keeps only the salvageable findings", () => { + expect(result.findings.map((f) => f.ruleId)).toEqual(["r-3", "r-4"]); + expect(result.counts.received).toBe(7); + expect(result.counts.dropped).toBe(5); + expect(result.findings.some((f) => f.location.file.includes("passwd"))).toBe(false); + expect(result.findings.some((f) => f.location.file.includes("home"))).toBe(false); + }); + + it("repairs recoverable fields and reports what it changed", () => { + const r3 = result.findings.find((f) => f.ruleId === "r-3")!; + expect(r3.severity).toBe(Severity.INFO); + expect(r3.message).toBe("r-3"); + expect(r3.location).toEqual({ file: "src/a.ts", startLine: 1, endLine: 1, startColumn: 9, endColumn: 9 }); + expect(r3.estimatedGasSavings).toBeUndefined(); + expect(result.findings.find((f) => f.ruleId === "r-4")!.location.startLine).toBe(7); + + const codes = result.issues.map((i) => i.code); + expect(codes).toEqual( + expect.arrayContaining([ + "MALFORMED_FINDING", + "MISSING_RULE_ID", + "INVALID_PATH", + "UNKNOWN_SEVERITY", + "INVALID_LINE", + "EMPTY_MESSAGE", + "INVALID_GAS_SAVINGS", + ]), + ); + }); + + it("reports unusable file errors without leaking the path", () => { + expect(result.errors).toEqual([{ file: "", message: "bad" }]); + }); +}); + +describe("failure semantics and options", () => { + it("returns an empty result with MALFORMED_RESULT for non-object input", () => { + for (const bad of [null, undefined, 5, "x", []]) { + const r = normalizeAnalysisResult(bad); + expect(r.findings).toEqual([]); + expect(r.issues[0].code).toBe("MALFORMED_RESULT"); + expect(r.analyzerVersion).toBe("unknown"); + } + }); + + it("flags a non-array findings field", () => { + const r = normalizeAnalysisResult({ findings: "oops" }); + expect(r.findings).toEqual([]); + expect(r.issues.map((i) => i.code)).toContain("MALFORMED_RESULT"); + }); + + it("truncates long messages and honours fallbackSeverity / maxIssues", () => { + const base = { ruleId: "r", location: { file: "a.ts", startLine: 1 } }; + const out = normalizeFindings( + [ + { ...base, message: "x".repeat(50), severity: "high" }, + { ...base, ruleId: "s", message: "m", severity: "???" }, + ], + { maxMessageLength: 10, fallbackSeverity: Severity.LOW, maxIssues: 1 }, + ); + expect(out.findings.find((f) => f.ruleId === "r")!.message).toHaveLength(10); + expect(out.findings.find((f) => f.ruleId === "s")!.severity).toBe(Severity.LOW); + expect(out.issues).toHaveLength(1); + expect(out.counts.issues).toBe(2); + }); + + it("strips rootDir from absolute paths when configured", () => { + const out = normalizeFindings( + [{ ruleId: "r", message: "m", severity: "low", location: { file: "/ws/job1/src/a.ts", startLine: 3 } }], + { rootDir: "/ws/job1" }, + ); + expect(out.findings[0].location.file).toBe("src/a.ts"); + }); +}); diff --git a/libs/engine/core/result-normalizer.ts b/libs/engine/core/result-normalizer.ts new file mode 100644 index 00000000..52c90205 --- /dev/null +++ b/libs/engine/core/result-normalizer.ts @@ -0,0 +1,448 @@ +import { createHash } from "crypto"; +import { Severity } from "./analyzer-interface"; +import type { AnalysisResult, Finding } from "./analyzer-interface"; + +/** + * Analyzer result normalization. + * + * Analyzers emit findings in slightly different shapes (path separators, + * severity spellings, 0/1-based or missing lines, duplicate reports). This + * module turns any analyzer output into one canonical, deterministic shape + * before it crosses the worker/persistence/reporting boundary. + * + * Guarantees: + * - Pure and tenant-agnostic: no I/O, no clock, no shared state. + * - Never throws. Unusable input degrades to dropped findings plus + * `issues`, so one bad finding never fails the whole scan. + * - Same findings (in any order) always produce byte-identical output. + * Nondeterministic data (timings, memory) lives only under `telemetry`. + */ + +export const NORMALIZATION_SCHEMA_VERSION = 1; +export const DEFAULT_MAX_MESSAGE_LENGTH = 2000; +export const DEFAULT_MAX_ISSUES = 100; + +export type NormalizationIssueCode = + | "MALFORMED_RESULT" + | "MALFORMED_FINDING" + | "MISSING_RULE_ID" + | "INVALID_PATH" + | "UNKNOWN_SEVERITY" + | "INVALID_LINE" + | "INVALID_COLUMN" + | "EMPTY_MESSAGE" + | "MESSAGE_TRUNCATED" + | "INVALID_GAS_SAVINGS" + | "DUPLICATE_FINDING"; + +export interface NormalizationIssue { + code: NormalizationIssueCode; + message: string; + ruleId?: string; + file?: string; + /** True when the finding was removed from the output because of this issue. */ + dropped: boolean; +} + +export interface NormalizationOptions { + /** Severity used when a finding's severity is missing or unrecognised. Default: info. */ + fallbackSeverity?: Severity; + /** Messages longer than this are truncated with an ellipsis. Default: 2000. */ + maxMessageLength?: number; + /** + * Absolute directory to strip from absolute finding paths (e.g. a checkout + * dir). Absolute paths outside it are rejected so host paths never leak. + */ + rootDir?: string; + /** Cap on recorded issues (counts stay exact). Default: 100. */ + maxIssues?: number; +} + +export interface NormalizedFinding { + /** Stable identity: sha256 of rule, location and message (first 32 hex chars). */ + key: string; + ruleId: string; + message: string; + severity: Severity; + location: { + file: string; + startLine: number; + endLine: number; + startColumn?: number; + endColumn?: number; + }; + estimatedGasSavings?: number; + suggestedFix?: Finding["suggestedFix"]; + metadata?: Record; +} + +export interface NormalizedAnalysisResult { + schemaVersion: number; + analyzerName?: string; + analyzerVersion: string; + filesAnalyzed: number; + findings: NormalizedFinding[]; + summary: AnalysisResult["summary"]; + totalEstimatedGasSavings?: number; + errors: Array<{ file: string; message: string }>; + issues: NormalizationIssue[]; + counts: { + received: number; + normalized: number; + dropped: number; + duplicates: number; + issues: number; + }; + /** Run-dependent data, kept apart so the rest of the result stays deterministic. */ + telemetry: { + analysisTimeMs?: number; + memoryUsage?: AnalysisResult["memoryUsage"]; + incremental?: AnalysisResult["incremental"]; + }; +} + +const SEVERITY_RANK: Record = { + [Severity.CRITICAL]: 5, + [Severity.HIGH]: 4, + [Severity.MEDIUM]: 3, + [Severity.LOW]: 2, + [Severity.INFO]: 1, +}; + +const SEVERITY_ALIASES: Record = { + critical: Severity.CRITICAL, + fatal: Severity.CRITICAL, + high: Severity.HIGH, + error: Severity.HIGH, + medium: Severity.MEDIUM, + moderate: Severity.MEDIUM, + warning: Severity.MEDIUM, + warn: Severity.MEDIUM, + low: Severity.LOW, + minor: Severity.LOW, + info: Severity.INFO, + informational: Severity.INFO, + note: Severity.INFO, + hint: Severity.INFO, +}; + +class IssueLog { + readonly issues: NormalizationIssue[] = []; + total = 0; + constructor(private readonly max: number) {} + add(issue: NormalizationIssue): void { + this.total++; + if (this.issues.length < this.max) this.issues.push(issue); + } +} + +function isRecord(v: unknown): v is Record { + return v !== null && typeof v === "object" && !Array.isArray(v); +} + +/** + * Canonical repository-relative path: forward slashes, no `.`/empty segments, + * `..` resolved. Returns undefined if the path is unusable or escapes the root. + */ +export function normalizeFindingPath( + raw: unknown, + rootDir?: string, +): string | undefined { + if (typeof raw !== "string") return undefined; + // eslint-disable-next-line no-control-regex + if (/[\u0000-\u001f\u007f]/.test(raw)) return undefined; + let p = raw.trim().replace(/\\/g, "/"); + if (p === "") return undefined; + + const isAbsolute = p.startsWith("/") || /^[A-Za-z]:\//.test(p); + if (isAbsolute) { + if (!rootDir) return undefined; + const root = rootDir.replace(/\\/g, "/").replace(/\/+$/, ""); + if (p !== root && !p.startsWith(`${root}/`)) return undefined; + p = p.slice(root.length); + } + + const out: string[] = []; + for (const seg of p.split("/")) { + if (seg === "" || seg === ".") continue; + if (seg === "..") { + if (out.length === 0) return undefined; + out.pop(); + } else { + out.push(seg); + } + } + return out.length > 0 ? out.join("/") : undefined; +} + +export function normalizeSeverity(raw: unknown): Severity | undefined { + if (typeof raw !== "string") return undefined; + return SEVERITY_ALIASES[raw.trim().toLowerCase()]; +} + +function toPositiveInt(raw: unknown): number | undefined { + if (typeof raw !== "number" || !Number.isFinite(raw)) return undefined; + const n = Math.floor(raw); + return n >= 1 ? n : undefined; +} + +/** Key-sorted, JSON-safe copy so metadata serialises identically every time. */ +function canonicalize(value: unknown, depth = 0): unknown { + if (value === null) return null; + if (typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") return Number.isFinite(value) ? value : undefined; + if (depth > 8) return undefined; + if (Array.isArray(value)) { + return value.map((v) => { + const c = canonicalize(v, depth + 1); + return c === undefined ? null : c; + }); + } + if (isRecord(value)) { + const out: Record = {}; + for (const k of Object.keys(value).sort()) { + const c = canonicalize(value[k], depth + 1); + if (c !== undefined) out[k] = c; + } + return out; + } + return undefined; +} + +function findingKey(f: Omit): string { + const l = f.location; + const material = [ + f.ruleId, + l.file, + l.startLine, + l.startColumn ?? "", + l.endLine, + l.endColumn ?? "", + f.message, + ].join("\u0000"); + return createHash("sha256").update(material).digest("hex").slice(0, 32); +} + +function normalizeFinding( + raw: unknown, + opts: Required> & + Pick, + log: IssueLog, +): NormalizedFinding | undefined { + if (!isRecord(raw)) { + log.add({ code: "MALFORMED_FINDING", message: "finding is not an object", dropped: true }); + return undefined; + } + + const ruleId = typeof raw.ruleId === "string" ? raw.ruleId.trim() : ""; + if (ruleId === "") { + log.add({ code: "MISSING_RULE_ID", message: "finding has no ruleId", dropped: true }); + return undefined; + } + + // Legacy shapes: `location.line`, or flat `file`/`line` fields. + const loc = isRecord(raw.location) ? raw.location : {}; + const rawFile = loc.file ?? raw.file ?? raw.filePath; + const file = normalizeFindingPath(rawFile, opts.rootDir); + if (file === undefined) { + log.add({ + code: "INVALID_PATH", + message: "finding path is missing, absolute, or escapes the repository root", + ruleId, + dropped: true, + }); + return undefined; + } + + let severity = normalizeSeverity(raw.severity); + if (severity === undefined) { + severity = opts.fallbackSeverity; + log.add({ + code: "UNKNOWN_SEVERITY", + message: `unrecognised severity, using ${severity}`, + ruleId, + file, + dropped: false, + }); + } + + let startLine = toPositiveInt(loc.startLine ?? loc.line ?? raw.line); + if (startLine === undefined) { + startLine = 1; + log.add({ code: "INVALID_LINE", message: "invalid startLine, using 1", ruleId, file, dropped: false }); + } + let endLine = toPositiveInt(loc.endLine); + if (endLine === undefined || endLine < startLine) endLine = startLine; + + let startColumn = toPositiveInt(loc.startColumn); + let endColumn = toPositiveInt(loc.endColumn); + if ((loc.startColumn !== undefined && startColumn === undefined) || + (loc.endColumn !== undefined && endColumn === undefined)) { + log.add({ code: "INVALID_COLUMN", message: "invalid column ignored", ruleId, file, dropped: false }); + } + if (startColumn === undefined) endColumn = undefined; + if (startColumn !== undefined && endColumn !== undefined && endLine === startLine && endColumn < startColumn) { + endColumn = startColumn; + } + + let message = typeof raw.message === "string" + ? raw.message.replace(/\r\n?/g, "\n").trim() + : ""; + if (message === "") { + message = ruleId; + log.add({ code: "EMPTY_MESSAGE", message: "empty message, using ruleId", ruleId, file, dropped: false }); + } else if (message.length > opts.maxMessageLength) { + message = `${message.slice(0, opts.maxMessageLength - 1)}…`; + log.add({ code: "MESSAGE_TRUNCATED", message: "message truncated", ruleId, file, dropped: false }); + } + + const location: NormalizedFinding["location"] = { file, startLine, endLine }; + if (startColumn !== undefined) location.startColumn = startColumn; + if (endColumn !== undefined) location.endColumn = endColumn; + + const normalized: Omit = { ruleId, message, severity, location }; + + if (raw.estimatedGasSavings !== undefined) { + const g = raw.estimatedGasSavings; + if (typeof g === "number" && Number.isFinite(g) && g >= 0) { + normalized.estimatedGasSavings = g; + } else { + log.add({ code: "INVALID_GAS_SAVINGS", message: "invalid estimatedGasSavings removed", ruleId, file, dropped: false }); + } + } + if (isRecord(raw.suggestedFix) && typeof raw.suggestedFix.description === "string") { + normalized.suggestedFix = canonicalize(raw.suggestedFix) as Finding["suggestedFix"]; + } + if (isRecord(raw.metadata)) { + const meta = canonicalize(raw.metadata) as Record; + if (Object.keys(meta).length > 0) normalized.metadata = meta; + } + + return { key: findingKey(normalized), ...normalized }; +} + +function compareFindings(a: NormalizedFinding, b: NormalizedFinding): number { + const cmp = (x: string | number, y: string | number) => (x < y ? -1 : x > y ? 1 : 0); + return ( + cmp(a.location.file, b.location.file) || + cmp(a.location.startLine, b.location.startLine) || + cmp(a.location.startColumn ?? 0, b.location.startColumn ?? 0) || + cmp(a.ruleId, b.ruleId) || + cmp(SEVERITY_RANK[b.severity], SEVERITY_RANK[a.severity]) || + cmp(a.key, b.key) + ); +} + +export function summarizeFindings( + findings: ReadonlyArray>, +): AnalysisResult["summary"] { + const summary = { critical: 0, high: 0, medium: 0, low: 0, info: 0 }; + for (const f of findings) summary[f.severity]++; + return summary; +} + +export function normalizeFindings( + rawFindings: unknown, + options: NormalizationOptions = {}, +): { findings: NormalizedFinding[]; issues: NormalizationIssue[]; counts: NormalizedAnalysisResult["counts"] } { + const log = new IssueLog(options.maxIssues ?? DEFAULT_MAX_ISSUES); + const opts = { + fallbackSeverity: options.fallbackSeverity ?? Severity.INFO, + maxMessageLength: options.maxMessageLength ?? DEFAULT_MAX_MESSAGE_LENGTH, + rootDir: options.rootDir, + }; + + const list: unknown[] = Array.isArray(rawFindings) ? rawFindings : []; + if (rawFindings !== undefined && !Array.isArray(rawFindings)) { + log.add({ code: "MALFORMED_RESULT", message: "findings is not an array", dropped: true }); + } + + const normalized: NormalizedFinding[] = []; + let dropped = 0; + for (const raw of list) { + const n = normalizeFinding(raw, opts, log); + if (n) normalized.push(n); + else dropped++; + } + + // Sort before deduping so the surviving duplicate never depends on input order. + normalized.sort(compareFindings); + const seen = new Set(); + const unique: NormalizedFinding[] = []; + let duplicates = 0; + for (const f of normalized) { + if (seen.has(f.key)) { + duplicates++; + log.add({ code: "DUPLICATE_FINDING", message: "duplicate finding removed", ruleId: f.ruleId, file: f.location.file, dropped: true }); + continue; + } + seen.add(f.key); + unique.push(f); + } + + return { + findings: unique, + issues: log.issues, + counts: { + received: list.length, + normalized: unique.length, + dropped, + duplicates, + issues: log.total, + }, + }; +} + +/** + * Normalizes a full analyzer result. `analyzerName` is optional context that + * is carried through verbatim. + */ +export function normalizeAnalysisResult( + raw: unknown, + options: NormalizationOptions & { analyzerName?: string } = {}, +): NormalizedAnalysisResult { + const record = isRecord(raw) ? raw : undefined; + const { findings, issues, counts } = normalizeFindings(record?.findings, options); + + if (!record) { + issues.unshift({ code: "MALFORMED_RESULT", message: "analyzer result is not an object", dropped: true }); + counts.issues++; + } + + const errors: NormalizedAnalysisResult["errors"] = []; + if (Array.isArray(record?.errors)) { + for (const e of record!.errors as unknown[]) { + if (!isRecord(e)) continue; + const file = normalizeFindingPath(e.file, options.rootDir) ?? ""; + const message = typeof e.message === "string" ? e.message.trim() : "unknown error"; + errors.push({ file, message }); + } + errors.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : a.message < b.message ? -1 : a.message > b.message ? 1 : 0)); + } + + const gas = findings.reduce((sum, f) => sum + (f.estimatedGasSavings ?? 0), 0); + const filesAnalyzed = record && typeof record.filesAnalyzed === "number" && Number.isFinite(record.filesAnalyzed) && record.filesAnalyzed >= 0 + ? Math.floor(record.filesAnalyzed) + : 0; + + const result: NormalizedAnalysisResult = { + schemaVersion: NORMALIZATION_SCHEMA_VERSION, + analyzerVersion: typeof record?.analyzerVersion === "string" ? record.analyzerVersion : "unknown", + filesAnalyzed, + findings, + summary: summarizeFindings(findings), + errors, + issues, + counts, + telemetry: {}, + }; + if (options.analyzerName !== undefined) result.analyzerName = options.analyzerName; + if (gas > 0) result.totalEstimatedGasSavings = gas; + + if (record) { + if (typeof record.analysisTime === "number") result.telemetry.analysisTimeMs = record.analysisTime; + if (isRecord(record.memoryUsage)) result.telemetry.memoryUsage = record.memoryUsage as unknown as AnalysisResult["memoryUsage"]; + if (isRecord(record.incremental)) result.telemetry.incremental = record.incremental as unknown as AnalysisResult["incremental"]; + } + return result; +} diff --git a/package.json b/package.json index c2ad11f8..1f6b578b 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "test:cov": "jest --coverage", "test:e2e": "jest --config ./apps/api/test/jest-e2e.json", "test:regression": "jest --testPathPattern='tests/regression/security/stellar/regression.spec' --no-coverage --verbose", + "test:load": "jest --testPathPattern=tests/load --no-coverage", + "load:report": "ts-node scripts/run-load-test.ts", "bench:gas-comparator": "ts-node tests/benchmarks/gas-comparator.ts tests/benchmarks/fixtures", "test:contracts": "hardhat test", "license:notices": "node scripts/generate-third-party-notices.mjs", diff --git a/scripts/run-load-test.ts b/scripts/run-load-test.ts new file mode 100644 index 00000000..8ac46682 --- /dev/null +++ b/scripts/run-load-test.ts @@ -0,0 +1,50 @@ +/** + * Runs the scan-pipeline reliability scenarios and prints a report. + * + * npm run load:report -- [--scenario ] [--seed ] [--scale <1-100>] + * + * Exits 1 when any invariant is violated or an SLO target is missed. + */ +import { runSimulation } from "../tests/load/scan-pipeline.simulator"; +import { buildScenario, DEFAULT_SEED, SCENARIO_NAMES, ScenarioName } from "../tests/load/scenarios"; +import { evaluateSlo, SLO_TARGETS } from "../tests/load/slo"; + +function arg(name: string): string | undefined { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? process.argv[i + 1] : undefined; +} + +const scenarioArg = arg("scenario"); +if (scenarioArg && !SCENARIO_NAMES.includes(scenarioArg as ScenarioName)) { + console.error(`unknown scenario "${scenarioArg}". Choose from: ${SCENARIO_NAMES.join(", ")}`); + process.exit(2); +} +const names = scenarioArg ? [scenarioArg as ScenarioName] : [...SCENARIO_NAMES]; +const seed = arg("seed") === undefined ? DEFAULT_SEED : Number(arg("seed")); +const scale = arg("scale") === undefined ? 1 : Number(arg("scale")); + +let failed = false; +for (const name of names) { + const report = runSimulation(buildScenario(name, seed, scale)); + const slo = evaluateSlo(report, SLO_TARGETS[name]); + failed = failed || !slo.passed; + console.log(`\n${slo.passed ? "PASS" : "FAIL"} ${name} (seed ${seed}, scale ${scale})`); + console.log( + ` scans ${report.uniqueScans} unique / ${report.submitted} submitted, ${report.completed} completed, ` + + `${report.deadLettered} dead-lettered, ${report.retries} retries`, + ); + console.log( + ` faults: ${report.crashes} crashes, ${report.stalls} stalls, ${report.fencedOut} fenced out, ` + + `${report.reaped} reclaimed, ${report.replayedItems} items replayed`, + ); + console.log( + ` wait ms: quiet p50 ${report.quietWait.p50} / p95 ${report.quietWait.p95} / p99 ${report.quietWait.p99}; ` + + `all p95 ${report.wait.p95}`, + ); + console.log( + ` queue: max depth ${report.maxQueueDepth}, age compliance ${report.queueAge.compliance}, ` + + `utilization ${report.workerUtilization}, throughput ${report.throughputPerSec}/s`, + ); + for (const f of slo.failures) console.log(` ! ${f}`); +} +process.exit(failed ? 1 : 0); diff --git a/tests/fixtures/location-mapping/README.md b/tests/fixtures/location-mapping/README.md new file mode 100644 index 00000000..17a03d8f --- /dev/null +++ b/tests/fixtures/location-mapping/README.md @@ -0,0 +1,11 @@ +# Location Mapping Fixtures + +One module (`runner.*.ts`) in four revisions. In `v1`, `return exec(cmd);` appears twice +(lines 9 and 14) with identical text but different surroundings, which is the case that +line-number or text-only matching gets wrong. + +- `v1` — baseline. Findings are anchored on lines 9 and 14. +- `v2` — code inserted above and between the functions, so both lines shift. Each finding + must follow its own call, not the look-alike. +- `v3` — the flagged line at 9 is edited in place; its surroundings are unchanged (`modified`). +- `v4-rewritten` — the flagged code no longer exists (`lost`). diff --git a/tests/fixtures/location-mapping/runner.v1.ts b/tests/fixtures/location-mapping/runner.v1.ts new file mode 100644 index 00000000..0f498438 --- /dev/null +++ b/tests/fixtures/location-mapping/runner.v1.ts @@ -0,0 +1,15 @@ +import { exec, execFile } from 'child_process'; + +export function ping(host: string) { + return execFile('ping', ['-c', '1', host]); +} + +export function lookup(name: string) { + const cmd = `nslookup ${name}`; + return exec(cmd); +} + +export function logTail(path: string) { + const cmd = `tail -n 20 ${path}`; + return exec(cmd); +} diff --git a/tests/fixtures/location-mapping/runner.v2.ts b/tests/fixtures/location-mapping/runner.v2.ts new file mode 100644 index 00000000..13f4cec5 --- /dev/null +++ b/tests/fixtures/location-mapping/runner.v2.ts @@ -0,0 +1,25 @@ +import { exec, execFile } from 'child_process'; +import { logger } from './logger'; + +/** + * Diagnostics helpers. + */ + +export function ping(host: string) { + logger.info('ping', host); + return execFile('ping', ['-c', '1', host]); +} + +export function traceroute(host: string) { + return execFile('traceroute', [host]); +} + +export function lookup(name: string) { + const cmd = `nslookup ${name}`; + return exec(cmd); +} + +export function logTail(path: string) { + const cmd = `tail -n 20 ${path}`; + return exec(cmd); +} diff --git a/tests/fixtures/location-mapping/runner.v3.ts b/tests/fixtures/location-mapping/runner.v3.ts new file mode 100644 index 00000000..a0753282 --- /dev/null +++ b/tests/fixtures/location-mapping/runner.v3.ts @@ -0,0 +1,15 @@ +import { exec, execFile } from 'child_process'; + +export function ping(host: string) { + return execFile('ping', ['-c', '1', host]); +} + +export function lookup(name: string) { + const cmd = `nslookup ${name}`; + return exec(cmd, { timeout: 1000 }); +} + +export function logTail(path: string) { + const cmd = `tail -n 20 ${path}`; + return exec(cmd); +} diff --git a/tests/fixtures/location-mapping/runner.v4-rewritten.ts b/tests/fixtures/location-mapping/runner.v4-rewritten.ts new file mode 100644 index 00000000..1220eaf6 --- /dev/null +++ b/tests/fixtures/location-mapping/runner.v4-rewritten.ts @@ -0,0 +1,6 @@ +// Everything was rewritten; the flagged code is gone. +import { execFile } from 'child_process'; + +export function lookup(name: string) { + return execFile('nslookup', [name]); +} diff --git a/tests/fixtures/result-normalization/README.md b/tests/fixtures/result-normalization/README.md new file mode 100644 index 00000000..43591199 --- /dev/null +++ b/tests/fixtures/result-normalization/README.md @@ -0,0 +1,9 @@ +# Result Normalization Fixtures + +- `realistic-mixed-output.json` — raw output as several analyzers emit it today: Windows + separators, `./` prefixes, severity aliases (`ERROR`, `warning`), inverted line ranges, + legacy flat `file`/`line` fields, near-duplicate findings, CRLF messages. +- `regression-malformed-findings.json` — regression case: findings that must be dropped + (no rule id, path traversal, absolute host path, non-objects) or repaired (line 0, + unknown severity, empty message, negative gas savings, fractional line) without + failing the whole result. diff --git a/tests/fixtures/result-normalization/realistic-mixed-output.json b/tests/fixtures/result-normalization/realistic-mixed-output.json new file mode 100644 index 00000000..a68e41f7 --- /dev/null +++ b/tests/fixtures/result-normalization/realistic-mixed-output.json @@ -0,0 +1,43 @@ +{ + "analyzerVersion": "1.4.0", + "filesAnalyzed": 4, + "analysisTime": 183, + "findings": [ + { + "ruleId": "cmd-001", + "message": "User input reaches child_process.exec", + "severity": "critical", + "location": { "file": "src\\services\\runner.ts", "startLine": 42, "endLine": 42, "startColumn": 5, "endColumn": 38 } + }, + { + "ruleId": "cmd-004", + "message": "Command built with template literal\r\nuse execFile with an argument array", + "severity": "ERROR", + "location": { "file": "./src/services/runner.ts", "startLine": 57, "endLine": 55 } + }, + { + "ruleId": "cmd-001", + "message": "User input reaches child_process.exec", + "severity": "high", + "location": { "file": "src/services/runner.ts", "startLine": 42, "endLine": 42, "startColumn": 5, "endColumn": 38 } + }, + { + "ruleId": "cmd-002", + "message": "shell: true enabled on dynamic input", + "severity": "warning", + "file": "scripts/deploy/../deploy.py", + "line": 9 + }, + { + "ruleId": "gas-007", + "message": "Storage read inside loop", + "severity": "medium", + "location": { "file": "contracts/Vault.sol", "startLine": 120, "endLine": 131 }, + "estimatedGasSavings": 2100, + "metadata": { "z": 1, "a": { "y": 2, "b": 3 } } + } + ], + "errors": [ + { "file": "docs\\broken.md", "message": "unsupported encoding" } + ] +} diff --git a/tests/fixtures/result-normalization/regression-malformed-findings.json b/tests/fixtures/result-normalization/regression-malformed-findings.json new file mode 100644 index 00000000..bba4c29e --- /dev/null +++ b/tests/fixtures/result-normalization/regression-malformed-findings.json @@ -0,0 +1,12 @@ +{ + "findings": [ + null, + "not-a-finding", + { "message": "no rule id", "severity": "high", "location": { "file": "a.ts", "startLine": 1 } }, + { "ruleId": "r-1", "message": "escapes the repo", "severity": "high", "location": { "file": "../../etc/passwd", "startLine": 1 } }, + { "ruleId": "r-2", "message": "host path leak", "severity": "high", "location": { "file": "/home/runner/work/app/src/a.ts", "startLine": 1 } }, + { "ruleId": "r-3", "message": "", "severity": "bogus", "location": { "file": "src/a.ts", "startLine": 0, "endLine": -4, "startColumn": 9, "endColumn": 2 }, "estimatedGasSavings": -50 }, + { "ruleId": "r-4", "message": "ok", "severity": "low", "location": { "file": "src/b.ts", "startLine": 7.9 } } + ], + "errors": [{ "file": "../outside.ts", "message": "bad" }, "junk"] +} diff --git a/tests/load/README.md b/tests/load/README.md new file mode 100644 index 00000000..107e91af --- /dev/null +++ b/tests/load/README.md @@ -0,0 +1,8 @@ +# Reliability load suite + +See [docs/RELIABILITY_LOAD_TESTING.md](../../docs/RELIABILITY_LOAD_TESTING.md). + +```bash +npm run test:load # specs (LOAD_SCALE=1..100 for longer runs) +npm run load:report -- --scenario chaos --seed 7 # human-readable report +``` diff --git a/tests/load/harness/harness.spec.ts b/tests/load/harness/harness.spec.ts new file mode 100644 index 00000000..b4d3516c --- /dev/null +++ b/tests/load/harness/harness.spec.ts @@ -0,0 +1,170 @@ +import { Rng } from "./prng"; +import { EventScheduler } from "./scheduler"; +import { percentile, summarize } from "./stats"; +import { validateSimulationConfig } from "../scan-pipeline.simulator"; +import { buildScenario } from "../scenarios"; + +describe("Rng", () => { + it("is reproducible from its seed and differs between seeds", () => { + const a = new Rng(42); + const b = new Rng(42); + const c = new Rng(43); + const seqA = Array.from({ length: 20 }, () => a.next()); + expect(Array.from({ length: 20 }, () => b.next())).toEqual(seqA); + expect(Array.from({ length: 20 }, () => c.next())).not.toEqual(seqA); + }); + + it("keeps next() within [0, 1) and int() within its inclusive bounds", () => { + const rng = new Rng(7); + for (let i = 0; i < 2_000; i += 1) { + const f = rng.next(); + expect(f).toBeGreaterThanOrEqual(0); + expect(f).toBeLessThan(1); + const n = rng.int(3, 5); + expect(n).toBeGreaterThanOrEqual(3); + expect(n).toBeLessThanOrEqual(5); + } + expect(rng.int(9, 9)).toBe(9); + }); + + it("handles the probability boundaries exactly", () => { + const rng = new Rng(1); + for (let i = 0; i < 200; i += 1) { + expect(rng.chance(0)).toBe(false); + expect(rng.chance(1)).toBe(true); + } + }); + + it("produces an exponential distribution with roughly the requested mean", () => { + const rng = new Rng(99); + const n = 20_000; + let sum = 0; + for (let i = 0; i < n; i += 1) sum += rng.exponential(100); + expect(sum / n).toBeGreaterThan(90); + expect(sum / n).toBeLessThan(110); + }); + + it("forks independent, reproducible streams", () => { + const root = new Rng(5); + const a1 = root.fork("a"); + root.next(); // drawing from the parent must not disturb a fork + const a2 = new Rng(5).fork("a"); + expect(a1.next()).toBe(a2.next()); + expect(new Rng(5).fork("a").next()).not.toBe(new Rng(5).fork("b").next()); + }); + + it("rejects invalid arguments", () => { + expect(() => new Rng(1.5)).toThrow(TypeError); + const rng = new Rng(1); + expect(() => rng.int(5, 1)).toThrow(RangeError); + expect(() => rng.int(1.5, 3)).toThrow(RangeError); + expect(() => rng.chance(1.01)).toThrow(RangeError); + expect(() => rng.chance(Number.NaN)).toThrow(RangeError); + expect(() => rng.exponential(0)).toThrow(RangeError); + expect(() => rng.exponential(Number.POSITIVE_INFINITY)).toThrow(RangeError); + }); +}); + +describe("EventScheduler", () => { + it("runs events in time order, and in scheduling order at the same instant", () => { + const s = new EventScheduler(); + const order: string[] = []; + s.schedule(30, () => order.push("c")); + s.schedule(10, () => order.push("a1")); + s.schedule(10, () => order.push("a2")); + s.schedule(20, () => order.push("b")); + while (s.step()); + expect(order).toEqual(["a1", "a2", "b", "c"]); + expect(s.now).toBe(30); + expect(s.processed).toBe(4); + }); + + it("lets events schedule further events", () => { + const s = new EventScheduler(); + const seen: number[] = []; + s.schedule(5, () => { + seen.push(s.now); + s.after(10, () => seen.push(s.now)); + }); + while (s.step()); + expect(seen).toEqual([5, 15]); + }); + + it("stays correctly ordered across many events", () => { + const s = new EventScheduler(); + const rng = new Rng(3); + const times: number[] = []; + for (let i = 0; i < 5_000; i += 1) { + const at = rng.int(0, 10_000); + s.schedule(at, () => times.push(s.now)); + } + while (s.step()); + expect(times).toHaveLength(5_000); + expect([...times].sort((x, y) => x - y)).toEqual(times); + }); + + it("returns false when empty and rejects bad times", () => { + const s = new EventScheduler(); + expect(s.step()).toBe(false); + s.schedule(10, () => undefined); + s.step(); + expect(() => s.schedule(5, () => undefined)).toThrow(RangeError); + expect(() => s.schedule(Number.NaN, () => undefined)).toThrow(TypeError); + }); +}); + +describe("stats", () => { + it("uses nearest-rank percentiles with correct boundaries", () => { + const data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + expect(percentile(data, 0)).toBe(1); + expect(percentile(data, 50)).toBe(5); + expect(percentile(data, 95)).toBe(10); + expect(percentile(data, 100)).toBe(10); + expect(percentile([42], 99)).toBe(42); + expect(percentile([], 50)).toBe(0); + expect(() => percentile(data, 101)).toThrow(RangeError); + expect(() => percentile(data, -1)).toThrow(RangeError); + }); + + it("summarizes unsorted samples and handles the empty case", () => { + expect(summarize([30, 10, 20])).toEqual({ count: 3, min: 10, mean: 20, p50: 20, p95: 30, p99: 30, max: 30 }); + expect(summarize([])).toEqual({ count: 0, min: 0, mean: 0, p50: 0, p95: 0, p99: 0, max: 0 }); + expect(() => summarize([1, Number.NaN])).toThrow(TypeError); + }); +}); + +describe("validateSimulationConfig", () => { + const valid = () => buildScenario("steady"); + + it("accepts every built-in scenario", () => { + for (const name of ["steady", "burst", "noisy-neighbor", "worker-crash", "duplicate-storm", "flaky-poison", "chaos"] as const) { + expect(() => validateSimulationConfig(buildScenario(name))).not.toThrow(); + } + }); + + it("rejects configs that would make results meaningless", () => { + const cases: Array<(c: ReturnType) => void> = [ + (c) => { c.workers = 0; }, + (c) => { c.service.maxMs = c.service.minMs - 1; }, + (c) => { c.service.checkpointEvery = c.service.items + 1; }, + (c) => { c.leaseTtlMs = 100; }, // shorter than two item durations + (c) => { c.faults.crashRate = 0.6; c.faults.stallRate = 0.6; }, + (c) => { c.faults.failRate = -0.1; }, + (c) => { c.faults.stallMs = c.leaseTtlMs; }, + (c) => { c.retry.multiplier = 0.5; }, + (c) => { c.tenants = []; }, + (c) => { c.tenants.push({ ...c.tenants[0] }); }, + ]; + for (const mutate of cases) { + const config = valid(); + mutate(config); + expect(() => validateSimulationConfig(config)).toThrow(RangeError); + } + }); + + it("rejects an invalid scenario scale", () => { + expect(() => buildScenario("steady", 1, 0)).toThrow(RangeError); + expect(() => buildScenario("steady", 1, 101)).toThrow(RangeError); + expect(() => buildScenario("steady", 1, 1.5)).toThrow(RangeError); + }); +}); diff --git a/tests/load/harness/prng.ts b/tests/load/harness/prng.ts new file mode 100644 index 00000000..f4bad63c --- /dev/null +++ b/tests/load/harness/prng.ts @@ -0,0 +1,66 @@ +/** + * Seeded pseudo-random numbers for reproducible load runs. + * + * Math.random is never used in the load suite: a failing run must be replayable + * from its seed alone. Independent streams (`fork`) keep one concern's draws + * (say, arrivals) from shifting another's (say, faults) when code changes. + */ + +function hashString(value: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < value.length; i += 1) { + hash ^= value.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash >>> 0; +} + +export class Rng { + private state: number; + + constructor(private readonly seed: number) { + if (!Number.isInteger(seed)) { + throw new TypeError("seed must be an integer"); + } + this.state = seed >>> 0; + } + + /** Uniform float in [0, 1) (mulberry32). */ + next(): number { + this.state = (this.state + 0x6d2b79f5) >>> 0; + let t = this.state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + } + + /** Uniform integer in [min, max], both inclusive. */ + int(min: number, max: number): number { + if (!Number.isInteger(min) || !Number.isInteger(max) || max < min) { + throw new RangeError("int() needs integers with min <= max"); + } + return min + Math.floor(this.next() * (max - min + 1)); + } + + /** True with probability `p` (0 never, 1 always). */ + chance(p: number): boolean { + if (!(p >= 0 && p <= 1)) { + throw new RangeError("probability must be within [0, 1]"); + } + return this.next() < p; + } + + /** Exponentially distributed value with the given mean (for Poisson gaps). */ + exponential(mean: number): number { + if (!(mean > 0) || !Number.isFinite(mean)) { + throw new RangeError("mean must be a positive finite number"); + } + return -mean * Math.log(1 - this.next()); + } + + /** An independent stream derived from this seed and a label. */ + fork(label: string): Rng { + const derived = hashString(`${this.seed}:${label}`); + return new Rng(derived === 0 ? 1 : derived | 0); + } +} diff --git a/tests/load/harness/scheduler.ts b/tests/load/harness/scheduler.ts new file mode 100644 index 00000000..787dce1c --- /dev/null +++ b/tests/load/harness/scheduler.ts @@ -0,0 +1,97 @@ +/** + * Discrete-event scheduler with a virtual clock. + * + * Load runs execute in simulated time: nothing sleeps, so minutes of queue + * behaviour run in milliseconds and give identical results on every machine. + * Events at the same instant run in the order they were scheduled. + */ + +interface Entry { + at: number; + seq: number; + run: () => void; +} + +export class EventScheduler { + private heap: Entry[] = []; + private seq = 0; + private current = 0; + private executed = 0; + + /** Current virtual time in milliseconds. */ + get now(): number { + return this.current; + } + + get pending(): number { + return this.heap.length; + } + + get processed(): number { + return this.executed; + } + + schedule(at: number, run: () => void): void { + if (!Number.isFinite(at)) { + throw new TypeError("event time must be a finite number"); + } + if (at < this.current) { + throw new RangeError(`cannot schedule in the past (${at} < ${this.current})`); + } + this.push({ at, seq: this.seq++, run }); + } + + after(delayMs: number, run: () => void): void { + this.schedule(this.current + delayMs, run); + } + + /** Runs the next event. Returns false when none remain. */ + step(): boolean { + const entry = this.pop(); + if (!entry) { + return false; + } + this.current = entry.at; + this.executed += 1; + entry.run(); + return true; + } + + private less(a: Entry, b: Entry): boolean { + return a.at < b.at || (a.at === b.at && a.seq < b.seq); + } + + private push(entry: Entry): void { + const heap = this.heap; + heap.push(entry); + let i = heap.length - 1; + while (i > 0) { + const parent = (i - 1) >> 1; + if (!this.less(heap[i], heap[parent])) break; + [heap[i], heap[parent]] = [heap[parent], heap[i]]; + i = parent; + } + } + + private pop(): Entry | undefined { + const heap = this.heap; + if (heap.length === 0) return undefined; + const top = heap[0]; + const last = heap.pop() as Entry; + if (heap.length > 0) { + heap[0] = last; + let i = 0; + for (;;) { + const l = 2 * i + 1; + const r = l + 1; + let min = i; + if (l < heap.length && this.less(heap[l], heap[min])) min = l; + if (r < heap.length && this.less(heap[r], heap[min])) min = r; + if (min === i) break; + [heap[i], heap[min]] = [heap[min], heap[i]]; + i = min; + } + } + return top; + } +} diff --git a/tests/load/harness/stats.ts b/tests/load/harness/stats.ts new file mode 100644 index 00000000..7488d5b6 --- /dev/null +++ b/tests/load/harness/stats.ts @@ -0,0 +1,40 @@ +export interface LatencySummary { + count: number; + min: number; + mean: number; + p50: number; + p95: number; + p99: number; + max: number; +} + +/** Nearest-rank percentile of an ascending array. Empty input gives 0. */ +export function percentile(sortedAscending: readonly number[], p: number): number { + if (!(p >= 0 && p <= 100)) { + throw new RangeError("percentile must be within [0, 100]"); + } + if (sortedAscending.length === 0) { + return 0; + } + const rank = Math.ceil((p / 100) * sortedAscending.length); + return sortedAscending[Math.min(sortedAscending.length, Math.max(1, rank)) - 1]; +} + +export function summarize(values: readonly number[]): LatencySummary { + for (const v of values) { + if (typeof v !== "number" || !Number.isFinite(v)) { + throw new TypeError("latency samples must be finite numbers"); + } + } + const sorted = [...values].sort((a, b) => a - b); + const total = sorted.reduce((sum, v) => sum + v, 0); + return { + count: sorted.length, + min: sorted.length ? sorted[0] : 0, + mean: sorted.length ? Math.round((total / sorted.length) * 1000) / 1000 : 0, + p50: percentile(sorted, 50), + p95: percentile(sorted, 95), + p99: percentile(sorted, 99), + max: sorted.length ? sorted[sorted.length - 1] : 0, + }; +} diff --git a/tests/load/scan-handler.load.spec.ts b/tests/load/scan-handler.load.spec.ts new file mode 100644 index 00000000..462e6100 --- /dev/null +++ b/tests/load/scan-handler.load.spec.ts @@ -0,0 +1,100 @@ +/** + * Integration load: many tenants submit scans to the real command-injection + * job handler at once, against one shared findings store. + */ +import * as fs from "fs"; +import * as path from "path"; +import { FindingsRepository } from "../../apps/api/src/findings/findings.repository"; +import { FindingsService } from "../../apps/api/src/findings/findings.service"; +import { + CommandInjectionScanJobHandler, + CommandInjectionScanJobPayload, +} from "../../apps/api/src/scanner/command-injection-scan.job"; + +const FIXTURE = path.resolve(__dirname, "../fixtures/command-injection/vulnerable-app"); + +function repoFiles(): Array<{ path: string; content: string }> { + const out: Array<{ path: string; content: string }> = []; + const walk = (dir: string) => { + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, e.name); + if (e.isDirectory()) walk(full); + else out.push({ path: path.relative(FIXTURE, full).split(path.sep).join("/"), content: fs.readFileSync(full, "utf8") }); + } + }; + walk(FIXTURE); + return out; +} + +const silent = { info: () => undefined, warn: () => undefined }; +const files = repoFiles(); + +const job = (org: string, repo: string, id: string): CommandInjectionScanJobPayload => ({ + organizationId: org, + repositoryId: repo, + analysisJobId: id, + files, +}); + +function setup() { + const findings = new FindingsService(new FindingsRepository()); + return { findings, handler: new CommandInjectionScanJobHandler({ findings, logger: silent }) }; +} + +jest.setTimeout(60_000); + +describe("scan job handler under concurrent multi-tenant load", () => { + const ORGS = 10; + const REPOS = 3; + + it("persists each tenant's findings exactly once and never leaks across tenants", async () => { + const baseline = await setup().handler.handle(job("baseline", "repo", "job")); + expect(baseline.findingsPersisted).toBeGreaterThan(0); + + const { handler, findings } = setup(); + const jobs: CommandInjectionScanJobPayload[] = []; + for (let o = 0; o < ORGS; o += 1) { + for (let r = 0; r < REPOS; r += 1) jobs.push(job(`org-${o}`, `repo-${r}`, `job-${o}-${r}`)); + } + const outcomes = await Promise.all(jobs.map((j) => handler.handle(j))); + + expect(outcomes.every((o) => o.status === "completed")).toBe(true); + expect(outcomes.every((o) => o.findingsPersisted === baseline.findingsPersisted)).toBe(true); + + for (let o = 0; o < ORGS; o += 1) { + for (let r = 0; r < REPOS; r += 1) { + const page = findings.list({ organizationId: `org-${o}`, repositoryId: `repo-${r}`, limit: 100 }); + expect(page.items).toHaveLength(baseline.findingsPersisted); + expect(page.items.every((f) => f.organizationId === `org-${o}` && f.repositoryId === `repo-${r}`)).toBe(true); + } + } + }); + + it("stays idempotent when the same scan is delivered many times at once", async () => { + const { handler, findings } = setup(); + const outcomes = await Promise.all(Array.from({ length: 20 }, () => handler.handle(job("org-a", "repo-1", "job-1")))); + + const persisted = outcomes.reduce((sum, o) => sum + o.findingsPersisted, 0); + const page = findings.list({ organizationId: "org-a", repositoryId: "repo-1", limit: 100 }); + expect(persisted).toBe(page.items.length); + expect(outcomes.filter((o) => o.findingsPersisted > 0)).toHaveLength(1); + }); + + it("isolates invalid jobs: they are rejected without affecting valid ones", async () => { + const { handler, findings } = setup(); + const settled = await Promise.allSettled([ + handler.handle(job("org-a", "repo-1", "ok-1")), + handler.handle(job("", "repo-1", "bad-org")), + handler.handle(job("org-b", "repo-1", "ok-2")), + handler.handle({ ...job("org-c", "repo-1", "bad-timeout"), timeoutMs: -1 }), + ]); + + expect(settled.map((s) => s.status)).toEqual(["fulfilled", "rejected", "fulfilled", "rejected"]); + for (const s of settled) { + if (s.status === "rejected") expect(s.reason).toEqual(expect.objectContaining({ code: "INVALID_JOB", retryable: false })); + } + expect(findings.list({ organizationId: "org-a", limit: 100 }).items.length).toBeGreaterThan(0); + expect(findings.list({ organizationId: "org-b", limit: 100 }).items.length).toBeGreaterThan(0); + expect(findings.list({ organizationId: "org-c", limit: 100 }).items).toHaveLength(0); + }); +}); diff --git a/tests/load/scan-pipeline.load.spec.ts b/tests/load/scan-pipeline.load.spec.ts new file mode 100644 index 00000000..ef4cd4e7 --- /dev/null +++ b/tests/load/scan-pipeline.load.spec.ts @@ -0,0 +1,178 @@ +/** + * Reliability load suite for the scan pipeline. + * + * Runs deterministic, virtual-time load against the real queue primitives + * (dedup, quotas, leases, checkpoints, queue-age SLO). The simulator reports + * broken invariants in `report.violations`; every scenario must finish with none. + * + * LOAD_SCALE=N (1-100) stretches each run N times longer. + */ + +import { runSimulation, LoadReport } from "./scan-pipeline.simulator"; +import { + buildScenario, + DEFAULT_SEED, + NOISY_REPOSITORY, + POISON_REPOSITORY, + SCENARIO_NAMES, + ScenarioName, +} from "./scenarios"; +import { evaluateSlo, SLO_TARGETS } from "./slo"; + +const SCALE = Math.min(100, Math.max(1, Math.floor(Number(process.env.LOAD_SCALE ?? 1)) || 1)); + +jest.setTimeout(120_000); + +const reports = new Map(); +function run(name: ScenarioName, seed = DEFAULT_SEED): LoadReport { + if (seed === DEFAULT_SEED && reports.has(name)) return reports.get(name) as LoadReport; + const report = runSimulation(buildScenario(name, seed, SCALE)); + if (seed === DEFAULT_SEED) reports.set(name, report); + return report; +} + +describe("every scenario", () => { + it.each(SCENARIO_NAMES)("%s: upholds all invariants and meets its SLO", (name) => { + const report = run(name); + expect(report.violations).toEqual([]); + expect(report.uniqueScans).toBeGreaterThan(100); + expect(report.completed + report.deadLettered).toBe(report.uniqueScans); + expect(report.submitted).toBe(report.uniqueScans + report.deduplicated); + expect(evaluateSlo(report, SLO_TARGETS[name]).failures).toEqual([]); + }); + + it.each(SCENARIO_NAMES)("%s: is deterministic for the same seed", (name) => { + const again = runSimulation(buildScenario(name, DEFAULT_SEED, SCALE)); + expect(again).toEqual(run(name)); + }); + + it("changes behaviour with a different seed", () => { + const other = run("steady", DEFAULT_SEED + 1); + expect(other.violations).toEqual([]); + expect(other.wait).not.toEqual(run("steady").wait); + }); +}); + +describe("steady load", () => { + it("completes everything without retries, faults or queue-age breaches", () => { + const r = run("steady"); + expect(r.completed).toBe(r.uniqueScans); + expect(r.deadLettered).toBe(0); + expect(r.retries).toBe(0); + expect(r.crashes + r.stalls + r.fencedOut + r.reaped).toBe(0); + expect(r.replayedItems).toBe(0); + expect(r.queueAge.breached).toBe(0); + expect(r.workerUtilization).toBeGreaterThan(0.1); + expect(r.workerUtilization).toBeLessThan(0.9); + expect(r.throughputPerSec).toBeGreaterThan(0.5); + }); + + it("never exceeds the per-repository concurrency limit", () => { + expect(run("steady").maxInFlightPerRepository).toBeLessThanOrEqual(2); + }); +}); + +describe("burst from one repository", () => { + it("is held back by the window quota while other tenants stay responsive", () => { + const r = run("burst"); + expect(r.quotaRejections.window).toBeGreaterThan(0); + expect(r.perTenant[NOISY_REPOSITORY].completed).toBe(r.perTenant[NOISY_REPOSITORY].submitted); + expect(r.maxQueueDepth).toBeGreaterThan(50); + expect(r.quietWait.p95).toBeLessThanOrEqual(SLO_TARGETS.burst.maxQuietWaitP95Ms); + // The burst tenant itself waits far longer than anyone else. + expect(r.perTenant[NOISY_REPOSITORY].wait.p95).toBeGreaterThan(r.quietWait.p95); + }); +}); + +describe("noisy neighbor", () => { + it("caps the noisy repository at its concurrency limit and protects quiet tenants", () => { + const r = run("noisy-neighbor"); + expect(r.perTenant[NOISY_REPOSITORY].maxInFlight).toBeLessThanOrEqual(2); + expect(r.quotaRejections.concurrency).toBeGreaterThan(0); + expect(r.quietWait.p95).toBeLessThanOrEqual(SLO_TARGETS["noisy-neighbor"].maxQuietWaitP95Ms); + expect(r.perTenant[NOISY_REPOSITORY].wait.p95).toBeGreaterThan(r.quietWait.p95 * 5); + // Quiet tenants lose nothing. + for (const [repo, t] of Object.entries(r.perTenant)) { + if (repo !== NOISY_REPOSITORY) expect(t.completed + t.deadLettered).toBe(t.submitted); + } + }); +}); + +describe("worker crashes and stalls", () => { + it("reclaims abandoned scans, fences stale workers and resumes from checkpoints", () => { + const r = run("worker-crash"); + expect(r.crashes).toBeGreaterThan(0); + expect(r.stalls).toBeGreaterThan(0); + // Every interrupted attempt is reclaimed exactly once... + expect(r.reaped).toBe(r.crashes + r.stalls); + // ...and every frozen worker that wakes up is refused by the lease manager. + expect(r.fencedOut).toBe(r.stalls); + // Checkpoint every 2 items: at most 2 items are redone per interruption. + expect(r.replayedItems).toBeGreaterThan(0); + expect(r.replayedItems).toBeLessThanOrEqual((r.crashes + r.stalls) * 2); + expect(r.completed).toBeGreaterThan(r.uniqueScans * 0.95); + }); +}); + +describe("duplicate submission storm", () => { + it("collapses re-sent revisions into a single scan each", () => { + const r = run("duplicate-storm"); + expect(r.deduplicated).toBeGreaterThan(r.uniqueScans * 0.5); + expect(r.submitted).toBe(r.uniqueScans + r.deduplicated); + expect(r.completed).toBe(r.uniqueScans); + expect(r.violations).toEqual([]); // includes "committed more than once" + }); +}); + +describe("retries and the dead-letter path", () => { + it("dead-letters a poison repository after exhausting retries without blocking others", () => { + const r = run("flaky-poison"); + const poison = r.perTenant[POISON_REPOSITORY]; + const poisonScans = poison.deadLettered + poison.completed; + expect(poison.completed).toBe(0); + expect(poison.deadLettered).toBeGreaterThan(20); + expect(poisonScans).toBeGreaterThan(0); + // 3 attempts per scan means 2 retries before giving up. + expect(r.retries).toBeGreaterThanOrEqual(poison.deadLettered * 2); + expect(r.completed).toBeGreaterThan(r.uniqueScans * 0.8); + expect(r.quietWait.p95).toBeLessThanOrEqual(SLO_TARGETS["flaky-poison"].maxQuietWaitP95Ms); + }); +}); + +describe("chaos", () => { + it.each([1, 2, 3, 4, 5])("holds every invariant with all faults combined (seed %i)", (seed) => { + const r = runSimulation(buildScenario("chaos", seed, SCALE)); + expect(r.violations).toEqual([]); + expect(r.completed + r.deadLettered).toBe(r.uniqueScans); + expect(r.crashes + r.stalls + r.retries).toBeGreaterThan(0); + expect(r.maxInFlightPerRepository).toBeLessThanOrEqual(2); + expect(r.fencedOut).toBe(r.stalls); + }); +}); + +describe("SLO evaluation", () => { + it("fails a degraded report and names what broke", () => { + const healthy = run("steady"); + const degraded: LoadReport = { + ...healthy, + violations: ["t=1ms something broke"], + completed: healthy.completed - 3, + quietWait: { ...healthy.quietWait, p95: 999_999 }, + deadLettered: 1, + queueAge: { ...healthy.queueAge, compliance: 0.5 }, + }; + const { passed, failures } = evaluateSlo(degraded, SLO_TARGETS.steady); + expect(passed).toBe(false); + expect(failures.join("\n")).toEqual( + expect.stringContaining("invariant violated: t=1ms something broke"), + ); + expect(failures.some((f) => f.includes("neither completed nor dead-lettered"))).toBe(true); + expect(failures.some((f) => f.includes("wait p95"))).toBe(true); + expect(failures.some((f) => f.includes("dead-letter ratio"))).toBe(true); + expect(failures.some((f) => f.includes("queue-age compliance"))).toBe(true); + }); + + it("passes a healthy report", () => { + expect(evaluateSlo(run("steady"), SLO_TARGETS.steady)).toEqual({ passed: true, failures: [] }); + }); +}); diff --git a/tests/load/scan-pipeline.simulator.ts b/tests/load/scan-pipeline.simulator.ts new file mode 100644 index 00000000..64882346 --- /dev/null +++ b/tests/load/scan-pipeline.simulator.ts @@ -0,0 +1,886 @@ +/** + * Scan pipeline load simulator. + * + * Drives the real reliability primitives of the scan queue (deduplication, + * per-repository quotas, leases with fencing tokens, checkpoints, queue-age + * SLO) from a pool of simulated workers, in virtual time, with injected faults. + * Only the clock, the job bodies and the faults are simulated; every admission, + * lease and checkpoint decision is made by production code. + * + * The simulator never throws on a broken invariant. It records a violation and + * keeps going, so one run reports everything that went wrong. Same config and + * seed always produce a byte-identical report. + */ + +import { QueueAgeSlo, QueueAgeSloConfig } from "../../apps/api/src/queue/queue-age-slo"; +import { ScanCheckpointStore } from "../../apps/api/src/queue/scan-checkpoint"; +import { ScanDeduplicator } from "../../apps/api/src/queue/scan-dedup"; +import { ScanLeaseManager } from "../../apps/api/src/queue/scan-lease"; +import { RepositoryScanQuota, ScanQuotaConfig } from "../../apps/api/src/queue/scan-quota"; +import { Rng } from "./harness/prng"; +import { EventScheduler } from "./harness/scheduler"; +import { LatencySummary, summarize } from "./harness/stats"; + +export interface TenantProfile { + repositoryId: string; + /** Mean Poisson arrival rate, scans per second. 0 disables steady arrivals. */ + ratePerSec: number; + /** `noisy` tenants are excluded from the "quiet tenant" latency figures. */ + role?: "normal" | "noisy"; + /** A one-off spike of scans. */ + burst?: { atMs: number; count: number; spreadMs: number }; + /** Every attempt fails: exercises retry exhaustion and the dead-letter path. */ + poison?: boolean; +} + +export interface SimulationConfig { + name: string; + seed: number; + workers: number; + /** New scans are accepted until this time; the run then drains. */ + arrivalWindowMs: number; + tenants: TenantProfile[]; + service: { + minMs: number; + maxMs: number; + /** Checkpointable units per scan. */ + items: number; + /** A checkpoint is saved every N completed items. */ + checkpointEvery: number; + }; + leaseTtlMs: number; + reaperIntervalMs: number; + sampleIntervalMs: number; + quota: Partial; + queueAge: QueueAgeSloConfig; + retry: { maxAttempts: number; backoffMs: number; multiplier: number }; + faults: { + /** Worker dies mid-scan; its lease must expire and the scan be reclaimed. */ + crashRate: number; + /** Worker freezes past its lease, then wakes as a "zombie" holding a stale token. */ + stallRate: number; + /** Scan throws; it is retried with exponential backoff. */ + failRate: number; + /** Chance a submission is re-sent, 1..maxDuplicates times. */ + duplicateRate: number; + maxDuplicates: number; + workerRestartMs: number; + stallMs: number; + }; + /** Safety valve against runaway runs. Default 5,000,000. */ + maxEvents?: number; + /** Check invariants after every event (default true). */ + checkInvariants?: boolean; +} + +export interface TenantReport { + submitted: number; + completed: number; + deadLettered: number; + wait: LatencySummary; + maxInFlight: number; +} + +export interface LoadReport { + scenario: string; + seed: number; + /** Virtual time at which the run fully drained. */ + drainedAtMs: number; + events: number; + submitted: number; + deduplicated: number; + uniqueScans: number; + completed: number; + deadLettered: number; + retries: number; + crashes: number; + stalls: number; + /** Stale-token workers refused by the lease manager. */ + fencedOut: number; + reaped: number; + quotaRejections: { concurrency: number; window: number }; + /** Items re-executed after an interruption (work lost between checkpoints). */ + replayedItems: number; + maxQueueDepth: number; + maxInFlightPerRepository: number; + workerUtilization: number; + throughputPerSec: number; + /** Submission to first start, all tenants. */ + wait: LatencySummary; + /** Submission to first start, `normal` tenants only. */ + quietWait: LatencySummary; + endToEnd: LatencySummary; + queueAge: { + samples: number; + healthy: number; + warning: number; + breached: number; + worstAgeMs: number; + compliance: number; + }; + perTenant: Record; + /** Broken invariants. A healthy run has none. */ + violations: string[]; +} + +interface Job { + scanId: string; + repositoryId: string; + arrivedAt: number; + queuedAt: number; + seq: number; + attempts: number; + itemMs: number; + poison: boolean; + firstStartedAt?: number; +} + +type FaultKind = "none" | "crash" | "stall" | "fail"; + +interface Attempt { + id: number; + job: Job; + worker: Worker; + token: number; + nextItem: number; + fault: FaultKind; + faultAtItem: number; + state: "running" | "crashed" | "stalled"; +} + +interface Worker { + id: string; + busy: boolean; + deadUntil: number; + busySince: number; +} + +const MAX_VIOLATIONS = 50; +const DEFAULT_MAX_EVENTS = 5_000_000; + +export function validateSimulationConfig(config: SimulationConfig): void { + const fail = (message: string): never => { + throw new RangeError(`invalid simulation config: ${message}`); + }; + const positiveInt = (value: number, field: string) => { + if (!Number.isInteger(value) || value <= 0) fail(`${field} must be a positive integer`); + }; + const rate = (value: number, field: string) => { + if (!(value >= 0 && value <= 1)) fail(`${field} must be within [0, 1]`); + }; + + positiveInt(config.workers, "workers"); + positiveInt(config.arrivalWindowMs, "arrivalWindowMs"); + positiveInt(config.service.items, "service.items"); + positiveInt(config.service.minMs, "service.minMs"); + positiveInt(config.leaseTtlMs, "leaseTtlMs"); + positiveInt(config.reaperIntervalMs, "reaperIntervalMs"); + positiveInt(config.sampleIntervalMs, "sampleIntervalMs"); + positiveInt(config.retry.maxAttempts, "retry.maxAttempts"); + if (!Number.isInteger(config.retry.backoffMs) || config.retry.backoffMs < 0) { + fail("retry.backoffMs must be a non-negative integer"); + } + if (!(config.retry.multiplier >= 1) || !Number.isFinite(config.retry.multiplier)) { + fail("retry.multiplier must be >= 1"); + } + if (config.service.maxMs < config.service.minMs) fail("service.maxMs must be >= service.minMs"); + if ( + !Number.isInteger(config.service.checkpointEvery) || + config.service.checkpointEvery < 1 || + config.service.checkpointEvery > config.service.items + ) { + fail("service.checkpointEvery must be between 1 and service.items"); + } + // A healthy worker renews once per item, so an item must fit well inside the TTL. + if (Math.ceil(config.service.maxMs / config.service.items) * 2 > config.leaseTtlMs) { + fail("leaseTtlMs must be at least twice the longest item duration"); + } + rate(config.faults.crashRate, "faults.crashRate"); + rate(config.faults.stallRate, "faults.stallRate"); + rate(config.faults.failRate, "faults.failRate"); + rate(config.faults.duplicateRate, "faults.duplicateRate"); + if (config.faults.crashRate + config.faults.stallRate + config.faults.failRate > 1) { + fail("crashRate + stallRate + failRate must not exceed 1"); + } + if (config.faults.stallMs <= config.leaseTtlMs) fail("faults.stallMs must exceed leaseTtlMs"); + if (config.tenants.length === 0) fail("at least one tenant is required"); + const seen = new Set(); + for (const tenant of config.tenants) { + if (!tenant.repositoryId) fail("tenant.repositoryId is required"); + if (seen.has(tenant.repositoryId)) fail(`duplicate tenant ${tenant.repositoryId}`); + seen.add(tenant.repositoryId); + if (!(tenant.ratePerSec >= 0) || !Number.isFinite(tenant.ratePerSec)) { + fail(`tenant ${tenant.repositoryId} ratePerSec must be >= 0`); + } + } +} + +export class ScanPipelineSimulation { + private readonly cfg: SimulationConfig; + private readonly sched = new EventScheduler(); + private readonly rng: Rng; + private readonly arrivalRng = new Map(); + private readonly serviceRng: Rng; + private readonly faultRng: Rng; + + private readonly dedup = new ScanDeduplicator(); + private readonly quota: RepositoryScanQuota; + private readonly quotaLimits: Required>; + private readonly leases: ScanLeaseManager; + private readonly checkpoints = new ScanCheckpointStore(); + private readonly slo: QueueAgeSlo; + + private readonly workers: Worker[]; + private readonly queues = new Map(); + private readonly running = new Map(); + private readonly latestToken = new Map(); + private readonly completions = new Map(); + private readonly executions = new Map(); + private readonly admissions = new Map(); + private readonly wakeAt = new Map(); + private readonly shaCounter = new Map(); + private readonly tenantByRepo = new Map(); + + private seq = 0; + private scanSeq = 0; + private attemptSeq = 0; + private queued = 0; + private arrivalsPending = 0; + private retryPending = 0; + private violationCount = 0; + private readonly violations: string[] = []; + + private submitted = 0; + private deduplicated = 0; + private uniqueScans = 0; + private completed = 0; + private retries = 0; + private crashes = 0; + private stalls = 0; + private failures = 0; + private fencedOut = 0; + private reaped = 0; + private maxQueueDepth = 0; + private busyMsTotal = 0; + private lastCompletionAt = 0; + private readonly rejections = { concurrency: 0, window: 0 }; + private readonly deadLetters: string[] = []; + private readonly deadLettersByRepo = new Map(); + private readonly submittedByRepo = new Map(); + private readonly completedByRepo = new Map(); + private readonly waitByRepo = new Map(); + private readonly maxInFlightByRepo = new Map(); + private readonly endToEnd: number[] = []; + private readonly ageSamples = { samples: 0, healthy: 0, warning: 0, breached: 0, worstAgeMs: 0 }; + + constructor(config: SimulationConfig) { + validateSimulationConfig(config); + this.cfg = config; + this.rng = new Rng(config.seed); + this.serviceRng = this.rng.fork("service"); + this.faultRng = this.rng.fork("faults"); + this.quota = new RepositoryScanQuota(config.quota); + this.quotaLimits = { + maxConcurrentPerRepository: 2, + maxScansPerWindow: 20, + windowMs: 60_000, + ...config.quota, + }; + this.leases = new ScanLeaseManager(config.leaseTtlMs, () => this.sched.now); + this.slo = new QueueAgeSlo(config.queueAge); + this.workers = Array.from({ length: config.workers }, (_, i) => ({ + id: `worker-${i}`, + busy: false, + deadUntil: 0, + busySince: 0, + })); + for (const tenant of config.tenants) { + this.queues.set(tenant.repositoryId, []); + this.tenantByRepo.set(tenant.repositoryId, tenant); + this.arrivalRng.set(tenant.repositoryId, this.rng.fork(`arrival:${tenant.repositoryId}`)); + this.waitByRepo.set(tenant.repositoryId, []); + this.maxInFlightByRepo.set(tenant.repositoryId, 0); + } + } + + run(): LoadReport { + for (const tenant of this.cfg.tenants) { + this.scheduleFirstArrival(tenant); + if (tenant.burst) this.scheduleBurst(tenant, tenant.burst); + } + this.sched.schedule(0, () => this.sample()); + this.sched.after(this.cfg.reaperIntervalMs, () => this.reap()); + + const maxEvents = this.cfg.maxEvents ?? DEFAULT_MAX_EVENTS; + const check = this.cfg.checkInvariants !== false; + while (this.sched.pending > 0) { + if (this.sched.processed >= maxEvents) { + this.violate(`event cap of ${maxEvents} reached before the run drained`); + break; + } + this.sched.step(); + if (check) this.checkInvariants(); + } + this.checkFinalState(); + return this.buildReport(); + } + + // ---- arrivals ----------------------------------------------------------- + + private scheduleFirstArrival(tenant: TenantProfile): void { + if (tenant.ratePerSec <= 0) return; + this.scheduleNextArrival(tenant, 0); + } + + private scheduleNextArrival(tenant: TenantProfile, from: number): void { + const gap = (this.arrivalRng.get(tenant.repositoryId) as Rng).exponential(1000 / tenant.ratePerSec); + const at = from + Math.max(1, Math.round(gap)); + if (at > this.cfg.arrivalWindowMs) return; + this.arrivalsPending += 1; + this.sched.schedule(at, () => { + this.arrivalsPending -= 1; + this.submit(tenant, this.nextSha(tenant.repositoryId)); + this.scheduleNextArrival(tenant, at); + }); + } + + private scheduleBurst(tenant: TenantProfile, burst: NonNullable): void { + const rng = this.arrivalRng.get(tenant.repositoryId) as Rng; + for (let i = 0; i < burst.count; i += 1) { + const at = burst.atMs + rng.int(0, Math.max(0, burst.spreadMs)); + this.arrivalsPending += 1; + this.sched.schedule(at, () => { + this.arrivalsPending -= 1; + this.submit(tenant, this.nextSha(tenant.repositoryId)); + }); + } + } + + private nextSha(repositoryId: string): string { + const n = (this.shaCounter.get(repositoryId) ?? 0) + 1; + this.shaCounter.set(repositoryId, n); + return n.toString(16).padStart(40, "0"); + } + + private submit(tenant: TenantProfile, sha: string): void { + const repo = tenant.repositoryId; + const now = this.sched.now; + this.submitted += 1; + this.submittedByRepo.set(repo, (this.submittedByRepo.get(repo) ?? 0) + 1); + + const scanId = `scan-${++this.scanSeq}`; + const outcome = this.dedup.register(repo, sha, scanId, now); + if (outcome.status === "duplicate") { + this.deduplicated += 1; + return; + } + + this.uniqueScans += 1; + const service = this.cfg.service; + const totalMs = this.serviceRng.int(service.minMs, service.maxMs); + this.enqueue({ + scanId, + repositoryId: repo, + arrivedAt: now, + queuedAt: now, + seq: 0, + attempts: 0, + itemMs: Math.max(1, Math.floor(totalMs / service.items)), + poison: tenant.poison === true, + }); + + // Clients re-send: the same revision arrives again shortly after. + const faults = this.cfg.faults; + if (faults.maxDuplicates > 0 && this.faultRng.chance(faults.duplicateRate)) { + const copies = this.faultRng.int(1, faults.maxDuplicates); + for (let i = 0; i < copies; i += 1) { + this.arrivalsPending += 1; + this.sched.after(Math.max(1, Math.round(this.faultRng.exponential(500))), () => { + this.arrivalsPending -= 1; + this.submit(tenant, sha); + }); + } + } + } + + // ---- queue and dispatch --------------------------------------------------- + + private enqueue(job: Job): void { + job.seq = ++this.seq; + job.queuedAt = this.sched.now; + (this.queues.get(job.repositoryId) as Job[]).push(job); + this.queued += 1; + this.maxQueueDepth = Math.max(this.maxQueueDepth, this.queued); + this.dispatch(); + } + + private idleWorker(): Worker | undefined { + const now = this.sched.now; + return this.workers.find((w) => !w.busy && now >= w.deadUntil); + } + + private dispatch(): void { + for (;;) { + const worker = this.idleWorker(); + if (!worker) return; + + // Oldest waiting job first, across repositories (global FIFO). + const heads: Job[] = []; + for (const queue of this.queues.values()) { + if (queue.length > 0) heads.push(queue[0]); + } + heads.sort((a, b) => a.seq - b.seq); + + let assigned = false; + for (const job of heads) { + const now = this.sched.now; + const decision = this.quota.tryAcquire(job.repositoryId, now); + if (!decision.allowed) { + if (decision.reason === "repository_window_exceeded") { + this.rejections.window += 1; + this.scheduleWake(job.repositoryId, now + (decision.retryAfterMs ?? 1)); + } else { + this.rejections.concurrency += 1; + } + continue; + } + this.recordAdmission(job.repositoryId, now); + + const lease = this.leases.acquire(job.scanId, worker.id); + if (!lease.granted) { + this.quota.release(job.repositoryId); + this.violate(`lease for ${job.scanId} still held by ${lease.heldBy} when it was re-dispatched`); + continue; + } + + (this.queues.get(job.repositoryId) as Job[]).shift(); + this.queued -= 1; + this.startAttempt(worker, job, lease.lease.token); + assigned = true; + break; + } + if (!assigned) return; + } + } + + private scheduleWake(repositoryId: string, at: number): void { + const existing = this.wakeAt.get(repositoryId); + if (existing !== undefined && existing >= this.sched.now && existing <= at) return; + this.wakeAt.set(repositoryId, at); + this.sched.schedule(at, () => { + if (this.wakeAt.get(repositoryId) === at) this.wakeAt.delete(repositoryId); + this.dispatch(); + }); + } + + private recordAdmission(repositoryId: string, now: number): void { + let log = this.admissions.get(repositoryId); + if (!log) { + log = []; + this.admissions.set(repositoryId, log); + } + log.push(now); + const { maxScansPerWindow, windowMs } = this.quotaLimits; + if (log.length > maxScansPerWindow) { + const earlier = log[log.length - 1 - maxScansPerWindow]; + if (now - earlier < windowMs) { + this.violate(`${repositoryId} admitted more than ${maxScansPerWindow} scans within ${windowMs}ms`); + } + } + } + + // ---- attempts --------------------------------------------------------------- + + private startAttempt(worker: Worker, job: Job, token: number): void { + const now = this.sched.now; + worker.busy = true; + worker.busySince = now; + const items = this.cfg.service.items; + + if (job.firstStartedAt === undefined) { + job.firstStartedAt = now; + (this.waitByRepo.get(job.repositoryId) as number[]).push(now - job.arrivedAt); + } + + const resume = this.checkpoints.resumeFrom(job.scanId); + const nextItem = resume ? resume.nextIndex : 0; + + const attempt: Attempt = { + id: ++this.attemptSeq, + job, + worker, + token, + nextItem, + fault: "none", + faultAtItem: -1, + state: "running", + }; + this.running.set(job.scanId, attempt); + this.latestToken.set(job.scanId, token); + const inFlight = this.countRunning(job.repositoryId); + this.maxInFlightByRepo.set(job.repositoryId, Math.max(this.maxInFlightByRepo.get(job.repositoryId) ?? 0, inFlight)); + + if (nextItem >= items) { + // Interrupted after the final checkpoint: nothing left but to commit. + this.complete(attempt); + return; + } + + this.chooseFault(attempt, items); + this.sched.after(job.itemMs, () => this.onItemDone(attempt)); + } + + private chooseFault(attempt: Attempt, items: number): void { + const lo = attempt.nextItem + 1; + const hi = items - 1; + if (hi < lo) return; // no room to fault mid-scan + if (attempt.job.poison) { + attempt.fault = "fail"; + attempt.faultAtItem = lo; + return; + } + const f = this.cfg.faults; + const u = this.faultRng.next(); + let kind: FaultKind = "none"; + if (u < f.crashRate) kind = "crash"; + else if (u < f.crashRate + f.stallRate) kind = "stall"; + else if (u < f.crashRate + f.stallRate + f.failRate) kind = "fail"; + if (kind === "none") return; + attempt.fault = kind; + attempt.faultAtItem = this.faultRng.int(lo, hi); + } + + private onItemDone(attempt: Attempt): void { + const { job, worker } = attempt; + const scanId = job.scanId; + const now = this.sched.now; + + // Heartbeat: a worker may only keep working while it holds the current lease. + const renewed = this.leases.renew(scanId, worker.id, attempt.token); + const isLatest = attempt.token === this.latestToken.get(scanId); + if (renewed && !isLatest) { + this.violate(`${scanId}: stale fencing token ${attempt.token} was allowed to renew`); + } + if (!renewed) { + this.fenceOut(attempt); + return; + } + + attempt.nextItem += 1; + this.executions.set(scanId, (this.executions.get(scanId) ?? 0) + 1); + + const items = this.cfg.service.items; + const finished = attempt.nextItem >= items; + const faultNow = !finished && attempt.faultAtItem === attempt.nextItem; + + if (faultNow && attempt.fault === "crash") { + // Dies before saving progress; the lease is left to expire. + attempt.state = "crashed"; + this.crashes += 1; + this.releaseWorkerTime(worker, now); + worker.busy = false; + worker.deadUntil = now + this.cfg.faults.workerRestartMs; + this.sched.schedule(worker.deadUntil, () => this.dispatch()); + return; + } + + try { + if (attempt.nextItem % this.cfg.service.checkpointEvery === 0 || finished) { + this.checkpoints.save(scanId, "analyze", attempt.nextItem, items, now); + } + } catch (error) { + this.violate(`${scanId}: checkpoint rejected valid progress (${(error as Error).message})`); + } + + if (finished) { + this.complete(attempt); + return; + } + if (faultNow && attempt.fault === "stall") { + attempt.state = "stalled"; + this.stalls += 1; + this.sched.after(this.cfg.faults.stallMs, () => this.onStallWake(attempt)); + return; + } + if (faultNow && attempt.fault === "fail") { + this.fail(attempt); + return; + } + this.sched.after(job.itemMs, () => this.onItemDone(attempt)); + } + + private onStallWake(attempt: Attempt): void { + const { job, worker } = attempt; + const renewed = this.leases.renew(job.scanId, worker.id, attempt.token); + if (renewed) { + this.violate(`${job.scanId}: a worker stalled past its lease TTL kept its lease`); + } + this.fenceOut(attempt); + } + + /** The lease manager refused this worker: abandon the scan without committing. */ + private fenceOut(attempt: Attempt): void { + const { job, worker } = attempt; + const live = this.leases.get(job.scanId); + if (live && live.owner === worker.id && live.token === attempt.token) { + this.violate(`${job.scanId}: lease manager refused the current, live holder`); + } + this.fencedOut += 1; + this.releaseWorkerTime(worker, this.sched.now); + worker.busy = false; + // Leave `running` alone: if nobody reclaimed it yet, the reaper will. + this.dispatch(); + } + + private complete(attempt: Attempt): void { + const { job, worker } = attempt; + const scanId = job.scanId; + const now = this.sched.now; + + if (!this.leases.release(scanId, worker.id, attempt.token)) { + this.violate(`${scanId}: completing worker could not release its own lease`); + } + this.quota.release(job.repositoryId); + this.running.delete(scanId); + + const count = (this.completions.get(scanId) ?? 0) + 1; + this.completions.set(scanId, count); + if (count > 1) this.violate(`${scanId}: committed ${count} times`); + if ((this.executions.get(scanId) ?? 0) < this.cfg.service.items) { + this.violate(`${scanId}: completed with fewer than ${this.cfg.service.items} items executed`); + } + + this.checkpoints.clear(scanId); + this.completed += 1; + this.completedByRepo.set(job.repositoryId, (this.completedByRepo.get(job.repositoryId) ?? 0) + 1); + this.endToEnd.push(now - job.arrivedAt); + this.lastCompletionAt = now; + this.releaseWorkerTime(worker, now); + worker.busy = false; + this.dispatch(); + } + + private fail(attempt: Attempt): void { + const { job, worker } = attempt; + this.failures += 1; + if (!this.leases.release(job.scanId, worker.id, attempt.token)) { + this.violate(`${job.scanId}: failing worker could not release its own lease`); + } + this.quota.release(job.repositoryId); + this.running.delete(job.scanId); + this.releaseWorkerTime(worker, this.sched.now); + worker.busy = false; + + job.attempts += 1; + this.retryOrDeadLetter(job, true); + this.dispatch(); + } + + private retryOrDeadLetter(job: Job, withBackoff: boolean): void { + const retry = this.cfg.retry; + if (job.attempts >= retry.maxAttempts) { + this.deadLetters.push(job.scanId); + this.deadLettersByRepo.set(job.repositoryId, (this.deadLettersByRepo.get(job.repositoryId) ?? 0) + 1); + this.checkpoints.clear(job.scanId); + return; + } + this.retries += 1; + if (!withBackoff) { + this.enqueue(job); + return; + } + const delay = Math.round(retry.backoffMs * Math.pow(retry.multiplier, job.attempts - 1)); + this.retryPending += 1; + this.sched.after(delay, () => { + this.retryPending -= 1; + this.enqueue(job); + }); + } + + // ---- background tasks ----------------------------------------------------- + + /** Reclaims scans whose worker died or froze: their lease has expired. */ + private reap(): void { + for (const attempt of [...this.running.values()]) { + const { job } = attempt; + if (this.leases.get(job.scanId) !== undefined) continue; + if (attempt.state === "running") { + this.violate(`${job.scanId}: lease of a healthy worker expired`); + } + this.running.delete(job.scanId); + this.quota.release(job.repositoryId); + this.reaped += 1; + job.attempts += 1; + this.retryOrDeadLetter(job, false); + } + this.leases.reclaimExpired(); + // Freed quota slots may unblock work that was waiting on them. + this.dispatch(); + if (!this.isDrained()) { + this.sched.after(this.cfg.reaperIntervalMs, () => this.reap()); + } + } + + private sample(): void { + const now = this.sched.now; + let oldest: number | undefined; + for (const queue of this.queues.values()) { + if (queue.length > 0 && (oldest === undefined || queue[0].queuedAt < oldest)) { + oldest = queue[0].queuedAt; + } + } + const age = oldest === undefined ? 0 : this.slo.evaluate(oldest, now); + const status = typeof age === "number" ? "healthy" : age.status; + const ageMs = typeof age === "number" ? 0 : age.ageMs; + this.ageSamples.samples += 1; + this.ageSamples[status] += 1; + this.ageSamples.worstAgeMs = Math.max(this.ageSamples.worstAgeMs, ageMs); + if (!this.isDrained()) { + this.sched.after(this.cfg.sampleIntervalMs, () => this.sample()); + } + } + + private isDrained(): boolean { + return this.arrivalsPending === 0 && this.retryPending === 0 && this.queued === 0 && this.running.size === 0; + } + + // ---- invariants ----------------------------------------------------------- + + private countRunning(repositoryId: string): number { + let n = 0; + for (const attempt of this.running.values()) { + if (attempt.job.repositoryId === repositoryId) n += 1; + } + return n; + } + + private checkInvariants(): void { + const now = this.sched.now; + const max = this.quotaLimits.maxConcurrentPerRepository; + for (const repo of this.queues.keys()) { + const inFlight = this.quota.snapshot(repo, now).inFlight; + const truth = this.countRunning(repo); + if (inFlight !== truth) { + this.violate(`${repo}: quota tracks ${inFlight} in flight but ${truth} scans are running`); + } + if (truth > max) { + this.violate(`${repo}: ${truth} concurrent scans exceed the limit of ${max}`); + } + } + } + + private checkFinalState(): void { + if (!this.isDrained()) { + this.violate("run ended with work still queued or running"); + } + if (this.completed + this.deadLetters.length !== this.uniqueScans) { + this.violate( + `scans lost: ${this.uniqueScans} unique, ${this.completed} completed, ${this.deadLetters.length} dead-lettered`, + ); + } + if (this.submitted - this.deduplicated !== this.uniqueScans) { + this.violate("deduplication accounting does not add up"); + } + if (this.leases.activeCount() !== 0) { + this.violate(`${this.leases.activeCount()} leases still live after the run drained`); + } + for (const repo of this.queues.keys()) { + if (this.quota.snapshot(repo, this.sched.now).inFlight !== 0) { + this.violate(`${repo}: quota slots leaked`); + } + } + // Every revision executed to completion exactly once, replays aside. + const replayBudget = (this.crashes + this.stalls + this.failures) * this.cfg.service.checkpointEvery; + if (this.replayedItems() > replayBudget) { + this.violate(`replayed ${this.replayedItems()} items, more than the ${replayBudget} the checkpoints allow`); + } + } + + private replayedItems(): number { + let replayed = 0; + for (const [scanId, count] of this.completions) { + if (count > 0) replayed += (this.executions.get(scanId) ?? 0) - this.cfg.service.items; + } + return replayed; + } + + private violate(message: string): void { + this.violationCount += 1; + if (this.violations.length < MAX_VIOLATIONS) { + this.violations.push(`t=${this.sched.now}ms ${message}`); + } + } + + private releaseWorkerTime(worker: Worker, now: number): void { + this.busyMsTotal += Math.max(0, now - worker.busySince); + } + + // ---- report ------------------------------------------------------------------- + + private buildReport(): LoadReport { + const drainedAtMs = this.sched.now; + const perTenant: Record = {}; + const quietWaits: number[] = []; + const allWaits: number[] = []; + let maxInFlightPerRepository = 0; + for (const tenant of this.cfg.tenants) { + const repo = tenant.repositoryId; + const waits = this.waitByRepo.get(repo) as number[]; + allWaits.push(...waits); + if ((tenant.role ?? "normal") === "normal") quietWaits.push(...waits); + const maxInFlight = this.maxInFlightByRepo.get(repo) ?? 0; + maxInFlightPerRepository = Math.max(maxInFlightPerRepository, maxInFlight); + perTenant[repo] = { + submitted: this.submittedByRepo.get(repo) ?? 0, + completed: this.completedByRepo.get(repo) ?? 0, + deadLettered: this.deadLettersByRepo.get(repo) ?? 0, + wait: summarize(waits), + maxInFlight, + }; + } + const makespan = Math.max(1, drainedAtMs); + const samples = this.ageSamples; + return { + scenario: this.cfg.name, + seed: this.cfg.seed, + drainedAtMs, + events: this.sched.processed, + submitted: this.submitted, + deduplicated: this.deduplicated, + uniqueScans: this.uniqueScans, + completed: this.completed, + deadLettered: this.deadLetters.length, + retries: this.retries, + crashes: this.crashes, + stalls: this.stalls, + fencedOut: this.fencedOut, + reaped: this.reaped, + quotaRejections: { ...this.rejections }, + replayedItems: this.replayedItems(), + maxQueueDepth: this.maxQueueDepth, + maxInFlightPerRepository, + workerUtilization: Math.round((this.busyMsTotal / (makespan * this.cfg.workers)) * 1000) / 1000, + throughputPerSec: Math.round((this.completed / (Math.max(1, this.lastCompletionAt) / 1000)) * 1000) / 1000, + wait: summarize(allWaits), + quietWait: summarize(quietWaits), + endToEnd: summarize(this.endToEnd), + queueAge: { + samples: samples.samples, + healthy: samples.healthy, + warning: samples.warning, + breached: samples.breached, + worstAgeMs: samples.worstAgeMs, + compliance: samples.samples === 0 ? 1 : Math.round((samples.healthy / samples.samples) * 1000) / 1000, + }, + perTenant, + violations: + this.violationCount > this.violations.length + ? [...this.violations, `... and ${this.violationCount - this.violations.length} more`] + : [...this.violations], + }; + } +} + +export function runSimulation(config: SimulationConfig): LoadReport { + return new ScanPipelineSimulation(config).run(); +} diff --git a/tests/load/scenarios.ts b/tests/load/scenarios.ts new file mode 100644 index 00000000..2f3b0aa7 --- /dev/null +++ b/tests/load/scenarios.ts @@ -0,0 +1,125 @@ +/** + * Named load scenarios for the scan pipeline. + * + * Each scenario is a plain SimulationConfig, so it can be tweaked, run from the + * CLI (`npm run load:report`) or imported by tests. `scale` stretches the + * arrival window (not the arrival rate), so utilisation stays comparable while + * the number of scans grows. At scale 1 each scenario submits roughly 700 to + * 1,500 scans and runs in well under a second. + */ + +import { SimulationConfig, TenantProfile } from "./scan-pipeline.simulator"; + +export const SCENARIO_NAMES = [ + "steady", + "burst", + "noisy-neighbor", + "worker-crash", + "duplicate-storm", + "flaky-poison", + "chaos", +] as const; + +export type ScenarioName = (typeof SCENARIO_NAMES)[number]; + +export const DEFAULT_SEED = 20240611; +export const MAX_SCALE = 100; + +const TENANT_COUNT = 12; +const BASE_RATE_PER_SEC = 0.1; + +/** repo-00 is the tenant scenarios turn into the noisy or bursting one. */ +export const NOISY_REPOSITORY = "repo-00"; +export const POISON_REPOSITORY = "repo-11"; + +function tenants(overrides: Record> = {}): TenantProfile[] { + return Array.from({ length: TENANT_COUNT }, (_, i) => { + const repositoryId = `repo-${String(i).padStart(2, "0")}`; + return { repositoryId, ratePerSec: BASE_RATE_PER_SEC, role: "normal", ...overrides[repositoryId] }; + }); +} + +function base(name: string, seed: number, scale: number): SimulationConfig { + return { + name, + seed, + workers: 8, + arrivalWindowMs: 600_000 * scale, + tenants: tenants(), + // 5 items per scan, checkpoint every 2: an interruption replays at most 2. + service: { minMs: 1_500, maxMs: 3_500, items: 5, checkpointEvery: 2 }, + leaseTtlMs: 15_000, + reaperIntervalMs: 5_000, + sampleIntervalMs: 5_000, + quota: { maxConcurrentPerRepository: 2, maxScansPerWindow: 60, windowMs: 60_000 }, + queueAge: { warningMs: 10_000, breachMs: 30_000 }, + retry: { maxAttempts: 3, backoffMs: 1_000, multiplier: 2 }, + faults: { + crashRate: 0, + stallRate: 0, + failRate: 0, + duplicateRate: 0, + maxDuplicates: 0, + workerRestartMs: 5_000, + stallMs: 30_000, + }, + }; +} + +export function buildScenario(name: ScenarioName, seed = DEFAULT_SEED, scale = 1): SimulationConfig { + if (!Number.isInteger(scale) || scale < 1 || scale > MAX_SCALE) { + throw new RangeError(`scale must be an integer between 1 and ${MAX_SCALE}`); + } + const cfg = base(name, seed, scale); + + switch (name) { + case "steady": + return cfg; + + case "burst": + // 150 scans from one repository in two seconds, against a tight window quota. + cfg.quota = { maxConcurrentPerRepository: 2, maxScansPerWindow: 10, windowMs: 30_000 }; + cfg.tenants = tenants({ + [NOISY_REPOSITORY]: { role: "noisy", burst: { atMs: 60_000, count: 150, spreadMs: 2_000 } }, + }); + return cfg; + + case "noisy-neighbor": + // One repository submits ~15x the others and exceeds what its quota allows. + cfg.tenants = tenants({ [NOISY_REPOSITORY]: { role: "noisy", ratePerSec: 1.5 } }); + return cfg; + + case "worker-crash": + cfg.faults = { ...cfg.faults, crashRate: 0.06, stallRate: 0.04 }; + return cfg; + + case "duplicate-storm": + cfg.faults = { ...cfg.faults, duplicateRate: 0.6, maxDuplicates: 4 }; + return cfg; + + case "flaky-poison": + // The poison repository fails on its first item every attempt (checkpoints + // are every 2 items, so it never gets far enough to save one). + cfg.faults = { ...cfg.faults, failRate: 0.1 }; + cfg.tenants = tenants({ [POISON_REPOSITORY]: { poison: true } }); + return cfg; + + case "chaos": + cfg.faults = { + ...cfg.faults, + crashRate: 0.04, + stallRate: 0.03, + failRate: 0.05, + duplicateRate: 0.3, + maxDuplicates: 3, + }; + cfg.tenants = tenants({ + [NOISY_REPOSITORY]: { role: "noisy", ratePerSec: 0.6, burst: { atMs: 120_000, count: 60, spreadMs: 3_000 } }, + [POISON_REPOSITORY]: { poison: true }, + }); + return cfg; + + default: + throw new RangeError(`unknown scenario: ${String(name)}`); + } +} diff --git a/tests/load/slo.ts b/tests/load/slo.ts new file mode 100644 index 00000000..e8953d0d --- /dev/null +++ b/tests/load/slo.ts @@ -0,0 +1,62 @@ +/** + * Reliability targets per scenario. + * + * Invariant violations are never tolerated. The latency and dead-letter targets + * are deliberately generous (several times what the scenarios typically + * produce) so they flag real regressions, such as a quota change that starves + * quiet tenants, rather than noise. Tighten them as the system improves. + */ + +import { LoadReport } from "./scan-pipeline.simulator"; +import { ScenarioName } from "./scenarios"; + +export interface SloTargets { + /** p95 of submission-to-start for tenants that are not noisy. */ + maxQuietWaitP95Ms: number; + /** Fraction of unique scans allowed to end in the dead-letter queue. */ + maxDeadLetterRatio: number; + /** Fraction of queue-age samples that must be healthy. Omit to skip. */ + minQueueAgeCompliance?: number; +} + +export const SLO_TARGETS: Record = { + steady: { maxQuietWaitP95Ms: 15_000, maxDeadLetterRatio: 0, minQueueAgeCompliance: 0.95 }, + burst: { maxQuietWaitP95Ms: 20_000, maxDeadLetterRatio: 0 }, + "noisy-neighbor": { maxQuietWaitP95Ms: 15_000, maxDeadLetterRatio: 0 }, + "worker-crash": { maxQuietWaitP95Ms: 30_000, maxDeadLetterRatio: 0.02 }, + "duplicate-storm": { maxQuietWaitP95Ms: 15_000, maxDeadLetterRatio: 0 }, + "flaky-poison": { maxQuietWaitP95Ms: 15_000, maxDeadLetterRatio: 0.15 }, + chaos: { maxQuietWaitP95Ms: 60_000, maxDeadLetterRatio: 0.15 }, +}; + +export interface SloEvaluation { + passed: boolean; + failures: string[]; +} + +export function evaluateSlo(report: LoadReport, targets: SloTargets): SloEvaluation { + const failures: string[] = []; + + for (const violation of report.violations) { + failures.push(`invariant violated: ${violation}`); + } + if (report.completed + report.deadLettered !== report.uniqueScans) { + failures.push( + `${report.uniqueScans - report.completed - report.deadLettered} scans neither completed nor dead-lettered`, + ); + } + if (report.quietWait.p95 > targets.maxQuietWaitP95Ms) { + failures.push(`quiet-tenant wait p95 ${report.quietWait.p95}ms exceeds ${targets.maxQuietWaitP95Ms}ms`); + } + const dlqRatio = report.uniqueScans === 0 ? 0 : report.deadLettered / report.uniqueScans; + if (dlqRatio > targets.maxDeadLetterRatio) { + failures.push(`dead-letter ratio ${dlqRatio.toFixed(3)} exceeds ${targets.maxDeadLetterRatio}`); + } + if ( + targets.minQueueAgeCompliance !== undefined && + report.queueAge.compliance < targets.minQueueAgeCompliance + ) { + failures.push(`queue-age compliance ${report.queueAge.compliance} is below ${targets.minQueueAgeCompliance}`); + } + return { passed: failures.length === 0, failures }; +}