diff --git a/apps/api/src/rules/performance-regression/__fixtures__/realistic-repo-fixture.json b/apps/api/src/rules/performance-regression/__fixtures__/realistic-repo-fixture.json new file mode 100644 index 00000000..39c2da7e --- /dev/null +++ b/apps/api/src/rules/performance-regression/__fixtures__/realistic-repo-fixture.json @@ -0,0 +1,54 @@ +{ + "description": "Realistic repository scan fixtures for Soroban AMM and Stellar DEX contracts", + "tenantId": "defi-protocol-labs", + "repositoryId": "soroban-amm-v2", + "baselineCommit": "a1b2c3d4e5f67890123456789abcdef012345678", + "currentCommit": "b2c3d4e5f67890123456789abcdef0123456789a", + "rulePackVersion": "1.5.0", + "baselineSamples": [ + { + "ruleId": "soroban-storage-rent-check", + "durationMs": 420, + "memoryBytes": 10485760, + "status": "ok", + "findingsCount": 2 + }, + { + "ruleId": "soroban-constant-propagation", + "durationMs": 150, + "memoryBytes": 4194304, + "status": "ok", + "findingsCount": 5 + }, + { + "ruleId": "soroban-cpu-cost-check", + "durationMs": 280, + "memoryBytes": 8388608, + "status": "ok", + "findingsCount": 1 + } + ], + "normalScanSamples": [ + { + "ruleId": "soroban-storage-rent-check", + "durationMs": 435, + "memoryBytes": 10600000, + "status": "ok", + "findingsCount": 2 + }, + { + "ruleId": "soroban-constant-propagation", + "durationMs": 155, + "memoryBytes": 4200000, + "status": "ok", + "findingsCount": 5 + }, + { + "ruleId": "soroban-cpu-cost-check", + "durationMs": 290, + "memoryBytes": 8500000, + "status": "ok", + "findingsCount": 1 + } + ] +} diff --git a/apps/api/src/rules/performance-regression/__fixtures__/regression-case-fixture.json b/apps/api/src/rules/performance-regression/__fixtures__/regression-case-fixture.json new file mode 100644 index 00000000..967e576d --- /dev/null +++ b/apps/api/src/rules/performance-regression/__fixtures__/regression-case-fixture.json @@ -0,0 +1,32 @@ +{ + "description": "Regression case fixture exhibiting +185% CPU execution duration and +200% memory consumption", + "tenantId": "defi-protocol-labs", + "repositoryId": "soroban-amm-v2", + "baselineCommit": "a1b2c3d4e5f67890123456789abcdef012345678", + "regressedCommit": "c3d4e5f67890123456789abcdef0123456789a1b", + "rulePackVersion": "1.5.0", + "regressedSamples": [ + { + "ruleId": "soroban-storage-rent-check", + "durationMs": 1200, + "memoryBytes": 31457280, + "status": "ok", + "findingsCount": 12 + }, + { + "ruleId": "soroban-constant-propagation", + "durationMs": 160, + "memoryBytes": 4300000, + "status": "ok", + "findingsCount": 5 + }, + { + "ruleId": "soroban-cpu-cost-check", + "durationMs": 310, + "memoryBytes": 8600000, + "status": "ok", + "findingsCount": 1 + } + ], + "expectedBreachedRules": ["soroban-storage-rent-check"] +} diff --git a/apps/api/src/rules/performance-regression/index.ts b/apps/api/src/rules/performance-regression/index.ts new file mode 100644 index 00000000..d28dd212 --- /dev/null +++ b/apps/api/src/rules/performance-regression/index.ts @@ -0,0 +1,7 @@ +export * from "./rule-performance-types"; +export * from "./rule-performance-config"; +export * from "./rule-performance-metrics"; +export * from "./rule-performance-evaluator"; +export * from "./rule-performance-store"; +export * from "./rule-performance.service"; +export * from "./rule-performance-worker"; diff --git a/apps/api/src/rules/performance-regression/rule-performance-config.spec.ts b/apps/api/src/rules/performance-regression/rule-performance-config.spec.ts new file mode 100644 index 00000000..b4743717 --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-config.spec.ts @@ -0,0 +1,110 @@ +import { + DEFAULT_RULE_PERFORMANCE_CONFIG, + loadRulePerformanceConfigFromEnv, + resolveRulePerformanceConfig, +} from "./rule-performance-config"; + +describe("RulePerformanceConfig", () => { + describe("resolveRulePerformanceConfig", () => { + it("returns default values when overrides are empty", () => { + const config = resolveRulePerformanceConfig(); + expect(config.enabled).toBe(true); + expect(config.defaultThresholds.maxDurationMs).toBe(5_000); + expect(config.defaultThresholds.maxDurationDeltaPercent).toBe(50); + expect(config.defaultThresholds.maxMemoryBytes).toBe(256 * 1024 * 1024); + expect(config.defaultThresholds.maxErrorRate).toBe(0.05); + expect(config.defaultThresholds.warningFactor).toBe(0.8); + expect(config.maxSamplesPerRule).toBe(1_000); + expect(config.baselineRetentionMs).toBe(30 * 24 * 60 * 60 * 1000); + }); + + it("merges valid overrides correctly", () => { + const config = resolveRulePerformanceConfig({ + enabled: false, + defaultThresholds: { + maxDurationMs: 3_000, + maxDurationDeltaPercent: 25, + }, + maxSamplesPerRule: 500, + }); + + expect(config.enabled).toBe(false); + expect(config.defaultThresholds.maxDurationMs).toBe(3_000); + expect(config.defaultThresholds.maxDurationDeltaPercent).toBe(25); + expect(config.defaultThresholds.maxMemoryBytes).toBe(256 * 1024 * 1024); + expect(config.maxSamplesPerRule).toBe(500); + }); + + it("rejects non-boolean enabled", () => { + expect(() => + resolveRulePerformanceConfig({ enabled: "true" as unknown as boolean }), + ).toThrow("enabled must be a boolean"); + }); + + it("rejects non-positive integers", () => { + expect(() => + resolveRulePerformanceConfig({ maxSamplesPerRule: 0 }), + ).toThrow("maxSamplesPerRule must be a positive integer"); + expect(() => + resolveRulePerformanceConfig({ defaultThresholds: { maxDurationMs: -10 } }), + ).toThrow("defaultThresholds.maxDurationMs must be a positive integer"); + }); + + it("rejects invalid error rate ratio", () => { + expect(() => + resolveRulePerformanceConfig({ defaultThresholds: { maxErrorRate: 1.5 } }), + ).toThrow("defaultThresholds.maxErrorRate must be a number between 0 and 1"); + expect(() => + resolveRulePerformanceConfig({ defaultThresholds: { maxErrorRate: -0.1 } }), + ).toThrow("defaultThresholds.maxErrorRate must be a number between 0 and 1"); + }); + + it("validates rule overrides", () => { + const config = resolveRulePerformanceConfig({ + ruleOverrides: { + "custom-rule": { maxDurationMs: 10_000 }, + }, + }); + expect(config.ruleOverrides["custom-rule"].maxDurationMs).toBe(10_000); + + expect(() => + resolveRulePerformanceConfig({ + ruleOverrides: { "": { maxDurationMs: 100 } }, + }), + ).toThrow("ruleOverrides keys must be non-empty strings"); + }); + }); + + describe("loadRulePerformanceConfigFromEnv", () => { + it("loads configuration from environment variables", () => { + const env: Record = { + RULE_PERFORMANCE_TRACKING_ENABLED: "false", + RULE_PERFORMANCE_DEFAULT_MAX_DURATION_MS: "2500", + RULE_PERFORMANCE_DEFAULT_MAX_DURATION_DELTA_PERCENT: "30", + RULE_PERFORMANCE_DEFAULT_MAX_MEMORY_BYTES: "1000000", + RULE_PERFORMANCE_DEFAULT_MAX_MEMORY_DELTA_PERCENT: "40", + RULE_PERFORMANCE_DEFAULT_MAX_ERROR_RATE: "0.02", + RULE_PERFORMANCE_WARNING_FACTOR: "0.75", + RULE_PERFORMANCE_MAX_SAMPLES_PER_RULE: "200", + }; + + const config = loadRulePerformanceConfigFromEnv(env); + expect(config.enabled).toBe(false); + expect(config.defaultThresholds.maxDurationMs).toBe(2500); + expect(config.defaultThresholds.maxDurationDeltaPercent).toBe(30); + expect(config.defaultThresholds.maxMemoryBytes).toBe(1000000); + expect(config.defaultThresholds.maxMemoryDeltaPercent).toBe(40); + expect(config.defaultThresholds.maxErrorRate).toBe(0.02); + expect(config.defaultThresholds.warningFactor).toBe(0.75); + expect(config.maxSamplesPerRule).toBe(200); + }); + + it("throws on unparseable numbers in env", () => { + expect(() => + loadRulePerformanceConfigFromEnv({ + RULE_PERFORMANCE_DEFAULT_MAX_DURATION_MS: "not-a-number", + }), + ).toThrow("RULE_PERFORMANCE_DEFAULT_MAX_DURATION_MS must be a number"); + }); + }); +}); diff --git a/apps/api/src/rules/performance-regression/rule-performance-config.ts b/apps/api/src/rules/performance-regression/rule-performance-config.ts new file mode 100644 index 00000000..2330a98c --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-config.ts @@ -0,0 +1,257 @@ +/** + * Configuration surface for tracking rule performance regression thresholds (#1251). + */ + +import { RulePerformanceThresholds } from "./rule-performance-types"; + +export interface RulePerformanceConfig { + /** Master switch and kill switch. When false, tracking & evaluations are no-ops. */ + enabled: boolean; + /** Global default thresholds applied to rules unless overridden. */ + defaultThresholds: RulePerformanceThresholds; + /** Per-rule threshold overrides keyed by ruleId. */ + ruleOverrides: Record; + /** Maximum number of execution samples stored or processed per rule. */ + maxSamplesPerRule: number; + /** Maximum duration allowed in ms before absolute rule failure. */ + maxAllowedDurationMs: number; + /** Maximum memory allowed in bytes before absolute rule failure. */ + maxAllowedMemoryBytes: number; + /** How long baselines are retained in milliseconds (default: 30 days). */ + baselineRetentionMs: number; + /** Maximum allowed tenant ID and repo ID string length. */ + maxIdentifierLength: number; +} + +export const DEFAULT_RULE_PERFORMANCE_THRESHOLDS: Readonly = + Object.freeze({ + maxDurationMs: 5_000, + maxDurationDeltaPercent: 50, // +50% regression from baseline + maxMemoryBytes: 256 * 1024 * 1024, // 256MB + maxMemoryDeltaPercent: 50, // +50% memory increase + maxErrorRate: 0.05, // 5% errors + warningFactor: 0.8, // 80% threshold triggers warning + }); + +export const DEFAULT_RULE_PERFORMANCE_CONFIG: Readonly = + Object.freeze({ + enabled: true, + defaultThresholds: DEFAULT_RULE_PERFORMANCE_THRESHOLDS, + ruleOverrides: {}, + maxSamplesPerRule: 1_000, + maxAllowedDurationMs: 60_000, + maxAllowedMemoryBytes: 1024 * 1024 * 1024, // 1GB + baselineRetentionMs: 30 * 24 * 60 * 60 * 1000, + maxIdentifierLength: 128, + }); + +const ENV_KEYS: Record = { + enabled: "RULE_PERFORMANCE_TRACKING_ENABLED", + maxDurationMs: "RULE_PERFORMANCE_DEFAULT_MAX_DURATION_MS", + maxDurationDeltaPercent: "RULE_PERFORMANCE_DEFAULT_MAX_DURATION_DELTA_PERCENT", + maxMemoryBytes: "RULE_PERFORMANCE_DEFAULT_MAX_MEMORY_BYTES", + maxMemoryDeltaPercent: "RULE_PERFORMANCE_DEFAULT_MAX_MEMORY_DELTA_PERCENT", + maxErrorRate: "RULE_PERFORMANCE_DEFAULT_MAX_ERROR_RATE", + warningFactor: "RULE_PERFORMANCE_WARNING_FACTOR", + maxSamplesPerRule: "RULE_PERFORMANCE_MAX_SAMPLES_PER_RULE", + maxAllowedDurationMs: "RULE_PERFORMANCE_MAX_ALLOWED_DURATION_MS", + maxAllowedMemoryBytes: "RULE_PERFORMANCE_MAX_ALLOWED_MEMORY_BYTES", + baselineRetentionMs: "RULE_PERFORMANCE_BASELINE_RETENTION_MS", +}; + +export function resolveRulePerformanceConfig( + overrides: Partial = {}, +): RulePerformanceConfig { + const defaultThresholds: RulePerformanceThresholds = { + ...DEFAULT_RULE_PERFORMANCE_THRESHOLDS, + ...(overrides.defaultThresholds || {}), + }; + + validateThresholds(defaultThresholds, "defaultThresholds"); + + const config: RulePerformanceConfig = { + ...DEFAULT_RULE_PERFORMANCE_CONFIG, + ...stripUndefined(overrides), + defaultThresholds, + ruleOverrides: overrides.ruleOverrides ?? {}, + }; + + if (typeof config.enabled !== "boolean") { + throw new RangeError("enabled must be a boolean"); + } + + requirePositiveInteger(config.maxSamplesPerRule, "maxSamplesPerRule"); + requirePositiveInteger(config.maxAllowedDurationMs, "maxAllowedDurationMs"); + requirePositiveInteger(config.maxAllowedMemoryBytes, "maxAllowedMemoryBytes"); + requirePositiveInteger(config.baselineRetentionMs, "baselineRetentionMs"); + requirePositiveInteger(config.maxIdentifierLength, "maxIdentifierLength"); + + for (const [ruleId, thresholds] of Object.entries(config.ruleOverrides)) { + if (!ruleId || ruleId.trim() === "") { + throw new RangeError("ruleOverrides keys must be non-empty strings"); + } + validateThresholds(thresholds, `ruleOverrides[${ruleId}]`); + } + + return config; +} + +export function loadRulePerformanceConfigFromEnv( + env: Record = process.env, +): RulePerformanceConfig { + const overrides: Partial = {}; + const defaultThresholds: Partial = {}; + + const enabledRaw = env[ENV_KEYS.enabled]; + if (enabledRaw !== undefined && enabledRaw.trim() !== "") { + overrides.enabled = parseBoolean(enabledRaw.trim(), ENV_KEYS.enabled); + } + + const durationRaw = env[ENV_KEYS.maxDurationMs]; + if (durationRaw !== undefined && durationRaw.trim() !== "") { + defaultThresholds.maxDurationMs = parseNumber(durationRaw.trim(), ENV_KEYS.maxDurationMs); + } + + const durationDeltaRaw = env[ENV_KEYS.maxDurationDeltaPercent]; + if (durationDeltaRaw !== undefined && durationDeltaRaw.trim() !== "") { + defaultThresholds.maxDurationDeltaPercent = parseNumber( + durationDeltaRaw.trim(), + ENV_KEYS.maxDurationDeltaPercent, + ); + } + + const memoryRaw = env[ENV_KEYS.maxMemoryBytes]; + if (memoryRaw !== undefined && memoryRaw.trim() !== "") { + defaultThresholds.maxMemoryBytes = parseNumber(memoryRaw.trim(), ENV_KEYS.maxMemoryBytes); + } + + const memoryDeltaRaw = env[ENV_KEYS.maxMemoryDeltaPercent]; + if (memoryDeltaRaw !== undefined && memoryDeltaRaw.trim() !== "") { + defaultThresholds.maxMemoryDeltaPercent = parseNumber( + memoryDeltaRaw.trim(), + ENV_KEYS.maxMemoryDeltaPercent, + ); + } + + const errorRateRaw = env[ENV_KEYS.maxErrorRate]; + if (errorRateRaw !== undefined && errorRateRaw.trim() !== "") { + defaultThresholds.maxErrorRate = parseNumber(errorRateRaw.trim(), ENV_KEYS.maxErrorRate); + } + + const warningFactorRaw = env[ENV_KEYS.warningFactor]; + if (warningFactorRaw !== undefined && warningFactorRaw.trim() !== "") { + defaultThresholds.warningFactor = parseNumber(warningFactorRaw.trim(), ENV_KEYS.warningFactor); + } + + const samplesRaw = env[ENV_KEYS.maxSamplesPerRule]; + if (samplesRaw !== undefined && samplesRaw.trim() !== "") { + overrides.maxSamplesPerRule = parseNumber(samplesRaw.trim(), ENV_KEYS.maxSamplesPerRule); + } + + const maxAllowedDurationRaw = env[ENV_KEYS.maxAllowedDurationMs]; + if (maxAllowedDurationRaw !== undefined && maxAllowedDurationRaw.trim() !== "") { + overrides.maxAllowedDurationMs = parseNumber( + maxAllowedDurationRaw.trim(), + ENV_KEYS.maxAllowedDurationMs, + ); + } + + const maxAllowedMemoryRaw = env[ENV_KEYS.maxAllowedMemoryBytes]; + if (maxAllowedMemoryRaw !== undefined && maxAllowedMemoryRaw.trim() !== "") { + overrides.maxAllowedMemoryBytes = parseNumber( + maxAllowedMemoryRaw.trim(), + ENV_KEYS.maxAllowedMemoryBytes, + ); + } + + const retentionRaw = env[ENV_KEYS.baselineRetentionMs]; + if (retentionRaw !== undefined && retentionRaw.trim() !== "") { + overrides.baselineRetentionMs = parseNumber( + retentionRaw.trim(), + ENV_KEYS.baselineRetentionMs, + ); + } + + if (Object.keys(defaultThresholds).length > 0) { + overrides.defaultThresholds = defaultThresholds; + } + + return resolveRulePerformanceConfig(overrides); +} + +function validateThresholds(thresholds: RulePerformanceThresholds, context: string): void { + if (thresholds.maxDurationMs !== undefined) { + requirePositiveInteger(thresholds.maxDurationMs, `${context}.maxDurationMs`); + } + if (thresholds.maxDurationDeltaPercent !== undefined) { + requireNonNegativeNumber( + thresholds.maxDurationDeltaPercent, + `${context}.maxDurationDeltaPercent`, + ); + } + if (thresholds.maxMemoryBytes !== undefined) { + requirePositiveInteger(thresholds.maxMemoryBytes, `${context}.maxMemoryBytes`); + } + if (thresholds.maxMemoryDeltaPercent !== undefined) { + requireNonNegativeNumber( + thresholds.maxMemoryDeltaPercent, + `${context}.maxMemoryDeltaPercent`, + ); + } + if (thresholds.maxErrorRate !== undefined) { + requireRatio(thresholds.maxErrorRate, `${context}.maxErrorRate`); + } + if (thresholds.warningFactor !== undefined) { + requireRatio(thresholds.warningFactor, `${context}.warningFactor`); + } +} + +function parseBoolean(value: string, envKey: string): boolean { + const lowered = value.toLowerCase(); + if (["true", "1", "yes", "on"].includes(lowered)) return true; + if (["false", "0", "no", "off"].includes(lowered)) return false; + throw new RangeError(`${envKey} must be a boolean`); +} + +function parseNumber(value: string, envKey: string): number { + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + throw new RangeError(`${envKey} must be a number`); + } + return parsed; +} + +function stripUndefined(value: T): Partial { + return Object.fromEntries( + Object.entries(value).filter(([, v]) => v !== undefined), + ) as Partial; +} + +function requirePositiveInteger(value: number, field: string): void { + if (!Number.isInteger(value) || value <= 0) { + throw new RangeError(`${field} must be a positive integer`); + } +} + +function requireNonNegativeInteger(value: number, field: string): void { + if (!Number.isInteger(value) || value < 0) { + throw new RangeError(`${field} must be a non-negative integer`); + } +} + +function requireNonNegativeNumber(value: number, field: string): void { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new RangeError(`${field} must be a non-negative number`); + } +} + +function requireRatio(value: number, field: string): void { + if ( + typeof value !== "number" || + !Number.isFinite(value) || + value < 0 || + value > 1 + ) { + throw new RangeError(`${field} must be a number between 0 and 1`); + } +} diff --git a/apps/api/src/rules/performance-regression/rule-performance-evaluator.spec.ts b/apps/api/src/rules/performance-regression/rule-performance-evaluator.spec.ts new file mode 100644 index 00000000..808f51a3 --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-evaluator.spec.ts @@ -0,0 +1,148 @@ +import { + evaluateRulePerformance, + evaluateSingleRule, + computeEvaluationFingerprint, +} from "./rule-performance-evaluator"; +import { + RuleExecutionSample, + RulePerformanceBaseline, + RulePerformanceThresholds, +} from "./rule-performance-types"; + +describe("RulePerformanceEvaluator", () => { + const defaultThresholds: RulePerformanceThresholds = { + maxDurationMs: 1000, + maxDurationDeltaPercent: 50, + maxMemoryBytes: 1000000, + maxMemoryDeltaPercent: 50, + maxErrorRate: 0.1, + warningFactor: 0.8, + }; + + const sampleBaseline: RulePerformanceBaseline = { + tenantId: "tenant-1", + repositoryId: "repo-1", + commitHash: "c001", + rulePackVersion: "1.0.0", + ruleId: "rule-a", + sampleCount: 10, + meanDurationMs: 200, + p95DurationMs: 250, + meanMemoryBytes: 200000, + p95MemoryBytes: 250000, + errorRate: 0, + createdAt: 1000, + updatedAt: 1000, + }; + + it("evaluates a rule as passed when metrics are well within limits", () => { + const samples: RuleExecutionSample[] = [ + { ruleId: "rule-a", durationMs: 210, memoryBytes: 210000, status: "ok", findingsCount: 0 }, + { ruleId: "rule-a", durationMs: 200, memoryBytes: 205000, status: "ok", findingsCount: 0 }, + ]; + + const result = evaluateSingleRule("rule-a", samples, sampleBaseline, defaultThresholds); + expect(result.status).toBe("passed"); + expect(result.breaches).toHaveLength(0); + expect(result.currentDurationMs).toBe(205); + }); + + it("detects a warning when duration approaches threshold", () => { + // 850ms is >= warning threshold (800ms) but < maxDurationMs (1000ms) + const samples: RuleExecutionSample[] = [ + { ruleId: "rule-standalone", durationMs: 850, status: "ok", findingsCount: 0 }, + ]; + + const result = evaluateSingleRule("rule-standalone", samples, undefined, defaultThresholds); + expect(result.status).toBe("warning"); + expect(result.breaches).toHaveLength(1); + expect(result.breaches[0].severity).toBe("warning"); + expect(result.breaches[0].metric).toBe("duration_ms"); + }); + + it("detects a breach when duration exceeds absolute threshold", () => { + const samples: RuleExecutionSample[] = [ + { ruleId: "rule-standalone", durationMs: 1200, status: "ok", findingsCount: 0 }, + ]; + + const result = evaluateSingleRule("rule-standalone", samples, undefined, defaultThresholds); + expect(result.status).toBe("breached"); + expect(result.breaches[0].severity).toBe("breach"); + }); + + it("detects a regression breach when duration increases by > 50% from baseline", () => { + // Baseline meanDurationMs is 200ms. 350ms is a +75% regression. + const samples: RuleExecutionSample[] = [ + { ruleId: "rule-a", durationMs: 350, memoryBytes: 200000, status: "ok", findingsCount: 0 }, + ]; + + const result = evaluateSingleRule("rule-a", samples, sampleBaseline, defaultThresholds); + expect(result.status).toBe("breached"); + const deltaBreach = result.breaches.find((b) => b.deltaPercent !== undefined); + expect(deltaBreach).toBeDefined(); + expect(deltaBreach?.deltaPercent).toBe(75); + expect(deltaBreach?.severity).toBe("breach"); + }); + + it("detects memory regression when memory consumption increases", () => { + // Baseline meanMemory is 200,000 bytes. 360,000 bytes is +80%. + const samples: RuleExecutionSample[] = [ + { ruleId: "rule-a", durationMs: 200, memoryBytes: 360000, status: "ok", findingsCount: 0 }, + ]; + + const result = evaluateSingleRule("rule-a", samples, sampleBaseline, defaultThresholds); + expect(result.status).toBe("breached"); + const memBreach = result.breaches.find((b) => b.metric === "memory_bytes"); + expect(memBreach).toBeDefined(); + expect(memBreach?.severity).toBe("breach"); + }); + + it("detects error rate breach", () => { + const samples: RuleExecutionSample[] = [ + { ruleId: "rule-flaky", durationMs: 100, status: "error", findingsCount: 0 }, + { ruleId: "rule-flaky", durationMs: 100, status: "ok", findingsCount: 0 }, + ]; // 50% error rate > 10% threshold + + const result = evaluateSingleRule("rule-flaky", samples, undefined, defaultThresholds); + expect(result.status).toBe("breached"); + expect(result.errorRate).toBe(0.5); + }); + + it("computes overall evaluation summary deterministically", () => { + const baselines = new Map(); + baselines.set("rule-a", sampleBaseline); + + const samples: RuleExecutionSample[] = [ + { ruleId: "rule-a", durationMs: 200, memoryBytes: 200000, status: "ok", findingsCount: 0 }, + { ruleId: "rule-b", durationMs: 1500, status: "ok", findingsCount: 0 }, // breached + ]; + + const summary1 = evaluateRulePerformance({ + tenantId: "tenant-1", + repositoryId: "repo-1", + commitHash: "commit-xyz", + rulePackVersion: "1.0.0", + samples, + baselines, + defaultThresholds, + clock: () => 1234567890, + }); + + const summary2 = evaluateRulePerformance({ + tenantId: "tenant-1", + repositoryId: "repo-1", + commitHash: "commit-xyz", + rulePackVersion: "1.0.0", + samples, + baselines, + defaultThresholds, + clock: () => 1234567890, + }); + + expect(summary1.status).toBe("breached"); + expect(summary1.totalRulesEvaluated).toBe(2); + expect(summary1.breachedCount).toBe(1); + expect(summary1.passedCount).toBe(1); + expect(summary1.fingerprint).toBe(summary2.fingerprint); + }); +}); diff --git a/apps/api/src/rules/performance-regression/rule-performance-evaluator.ts b/apps/api/src/rules/performance-regression/rule-performance-evaluator.ts new file mode 100644 index 00000000..d8e40de0 --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-evaluator.ts @@ -0,0 +1,355 @@ +/** + * Pure evaluation engine for rule performance regression thresholds (#1251). + * + * Deterministic: pure functions with no side-effects or system clock dependencies. + * Given the same samples, baseline, and thresholds, results are 100% reproducible. + */ + +import { + EvaluationSummary, + PerformanceRegressionStatus, + RegressionMetricKind, + RuleExecutionSample, + RuleMetricBreach, + RulePerformanceBaseline, + RulePerformanceThresholds, + RuleRegressionEvaluation, +} from "./rule-performance-types"; + +export interface EvaluationInput { + tenantId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + samples: RuleExecutionSample[]; + baselines?: Map; + defaultThresholds: RulePerformanceThresholds; + ruleOverrides?: Record; + clock?: () => number; +} + +export function evaluateRulePerformance( + input: EvaluationInput, +): EvaluationSummary { + const { + tenantId, + repositoryId, + commitHash, + rulePackVersion, + samples, + baselines = new Map(), + defaultThresholds, + ruleOverrides = {}, + clock = () => Date.now(), + } = input; + + // Group samples by ruleId + const samplesByRule = new Map(); + for (const sample of samples) { + if (!samplesByRule.has(sample.ruleId)) { + samplesByRule.set(sample.ruleId, []); + } + samplesByRule.get(sample.ruleId)!.push(sample); + } + + // Sort rule IDs for deterministic iteration + const sortedRuleIds = Array.from(samplesByRule.keys()).sort(); + + const ruleEvaluations: RuleRegressionEvaluation[] = []; + let passedCount = 0; + let warningCount = 0; + let breachedCount = 0; + + for (const ruleId of sortedRuleIds) { + const ruleSamples = samplesByRule.get(ruleId)!; + const baseline = baselines.get(ruleId); + const thresholds = { + ...defaultThresholds, + ...(ruleOverrides[ruleId] || {}), + }; + + const evaluation = evaluateSingleRule(ruleId, ruleSamples, baseline, thresholds); + ruleEvaluations.push(evaluation); + + if (evaluation.status === "breached") { + breachedCount++; + } else if (evaluation.status === "warning") { + warningCount++; + } else { + passedCount++; + } + } + + const overallStatus: PerformanceRegressionStatus = + breachedCount > 0 ? "breached" : warningCount > 0 ? "warning" : "passed"; + + const fingerprint = computeEvaluationFingerprint( + tenantId, + repositoryId, + commitHash, + rulePackVersion, + ruleEvaluations, + ); + + return { + tenantId, + repositoryId, + commitHash, + rulePackVersion, + status: overallStatus, + totalRulesEvaluated: sortedRuleIds.length, + passedCount, + warningCount, + breachedCount, + skippedRulesCount: 0, + ruleEvaluations, + evaluatedAt: clock(), + fingerprint, + }; +} + +export function evaluateSingleRule( + ruleId: string, + samples: RuleExecutionSample[], + baseline: RulePerformanceBaseline | undefined, + thresholds: RulePerformanceThresholds, +): RuleRegressionEvaluation { + if (samples.length === 0) { + return { + ruleId, + status: "passed", + breaches: [], + currentDurationMs: 0, + baselineDurationMs: baseline?.meanDurationMs, + currentMemoryBytes: 0, + baselineMemoryBytes: baseline?.meanMemoryBytes, + errorRate: 0, + }; + } + + const totalDuration = samples.reduce((sum, s) => sum + s.durationMs, 0); + const avgDuration = totalDuration / samples.length; + + const memorySamples = samples + .map((s) => s.memoryBytes) + .filter((m): m is number => m !== undefined && Number.isFinite(m)); + const avgMemory = + memorySamples.length > 0 + ? memorySamples.reduce((sum, m) => sum + m, 0) / memorySamples.length + : 0; + + const errorCount = samples.filter((s) => s.status !== "ok").length; + const errorRate = errorCount / samples.length; + + const breaches: RuleMetricBreach[] = []; + const warningFactor = thresholds.warningFactor ?? 0.8; + + // 1. Duration Absolute Threshold Check + if (thresholds.maxDurationMs !== undefined) { + checkMetricThreshold({ + metric: "duration_ms", + currentValue: avgDuration, + thresholdValue: thresholds.maxDurationMs, + warningFactor, + unit: "ms", + ruleId, + breaches, + }); + } + + // 2. Duration Delta vs Baseline Check + if ( + baseline && + baseline.meanDurationMs > 0 && + thresholds.maxDurationDeltaPercent !== undefined + ) { + const deltaPercent = + ((avgDuration - baseline.meanDurationMs) / baseline.meanDurationMs) * 100; + if (deltaPercent > 0) { + checkDeltaThreshold({ + metric: "duration_ms", + currentValue: avgDuration, + baselineValue: baseline.meanDurationMs, + deltaPercent, + thresholdPercent: thresholds.maxDurationDeltaPercent, + warningFactor, + ruleId, + breaches, + }); + } + } + + // 3. Memory Absolute Threshold Check + if (avgMemory > 0 && thresholds.maxMemoryBytes !== undefined) { + checkMetricThreshold({ + metric: "memory_bytes", + currentValue: avgMemory, + thresholdValue: thresholds.maxMemoryBytes, + warningFactor, + unit: "bytes", + ruleId, + breaches, + }); + } + + // 4. Memory Delta vs Baseline Check + if ( + baseline && + avgMemory > 0 && + baseline.meanMemoryBytes > 0 && + thresholds.maxMemoryDeltaPercent !== undefined + ) { + const deltaPercent = + ((avgMemory - baseline.meanMemoryBytes) / baseline.meanMemoryBytes) * 100; + if (deltaPercent > 0) { + checkDeltaThreshold({ + metric: "memory_bytes", + currentValue: avgMemory, + baselineValue: baseline.meanMemoryBytes, + deltaPercent, + thresholdPercent: thresholds.maxMemoryDeltaPercent, + warningFactor, + ruleId, + breaches, + }); + } + } + + // 5. Error Rate Threshold Check + if (thresholds.maxErrorRate !== undefined) { + checkMetricThreshold({ + metric: "error_rate", + currentValue: errorRate, + thresholdValue: thresholds.maxErrorRate, + warningFactor, + unit: "ratio", + ruleId, + breaches, + }); + } + + const hasBreach = breaches.some((b) => b.severity === "breach"); + const hasWarning = breaches.some((b) => b.severity === "warning"); + + const status: PerformanceRegressionStatus = hasBreach + ? "breached" + : hasWarning + ? "warning" + : "passed"; + + return { + ruleId, + status, + breaches, + currentDurationMs: Math.round(avgDuration * 100) / 100, + baselineDurationMs: baseline ? Math.round(baseline.meanDurationMs * 100) / 100 : undefined, + currentMemoryBytes: avgMemory > 0 ? Math.round(avgMemory) : undefined, + baselineMemoryBytes: + baseline && baseline.meanMemoryBytes > 0 + ? Math.round(baseline.meanMemoryBytes) + : undefined, + errorRate: Math.round(errorRate * 1000) / 1000, + }; +} + +function checkMetricThreshold(options: { + metric: RegressionMetricKind; + currentValue: number; + thresholdValue: number; + warningFactor: number; + unit: string; + ruleId: string; + breaches: RuleMetricBreach[]; +}): void { + const { metric, currentValue, thresholdValue, warningFactor, unit, ruleId, breaches } = + options; + const warningThreshold = thresholdValue * warningFactor; + + if (currentValue >= thresholdValue) { + breaches.push({ + metric, + severity: "breach", + currentValue: Math.round(currentValue * 100) / 100, + thresholdValue, + message: `Rule '${ruleId}' ${metric} (${roundVal(currentValue)}${unit}) breached threshold (${thresholdValue}${unit})`, + }); + } else if (currentValue >= warningThreshold) { + breaches.push({ + metric, + severity: "warning", + currentValue: Math.round(currentValue * 100) / 100, + thresholdValue, + message: `Rule '${ruleId}' ${metric} (${roundVal(currentValue)}${unit}) exceeded warning threshold (${roundVal(warningThreshold)}${unit})`, + }); + } +} + +function checkDeltaThreshold(options: { + metric: RegressionMetricKind; + currentValue: number; + baselineValue: number; + deltaPercent: number; + thresholdPercent: number; + warningFactor: number; + ruleId: string; + breaches: RuleMetricBreach[]; +}): void { + const { + metric, + currentValue, + baselineValue, + deltaPercent, + thresholdPercent, + warningFactor, + ruleId, + breaches, + } = options; + const warningPercent = thresholdPercent * warningFactor; + + if (deltaPercent >= thresholdPercent) { + breaches.push({ + metric, + severity: "breach", + currentValue: Math.round(currentValue * 100) / 100, + baselineValue: Math.round(baselineValue * 100) / 100, + thresholdValue: thresholdPercent, + deltaPercent: Math.round(deltaPercent * 10) / 10, + message: `Rule '${ruleId}' ${metric} regressed by +${roundVal(deltaPercent)}% from baseline (${roundVal(baselineValue)} -> ${roundVal(currentValue)}), breaching threshold (+${thresholdPercent}%)`, + }); + } else if (deltaPercent >= warningPercent) { + breaches.push({ + metric, + severity: "warning", + currentValue: Math.round(currentValue * 100) / 100, + baselineValue: Math.round(baselineValue * 100) / 100, + thresholdValue: thresholdPercent, + deltaPercent: Math.round(deltaPercent * 10) / 10, + message: `Rule '${ruleId}' ${metric} regressed by +${roundVal(deltaPercent)}% from baseline, approaching warning limit (+${roundVal(warningPercent)}%)`, + }); + } +} + +function roundVal(n: number): number { + return Math.round(n * 100) / 100; +} + +export function computeEvaluationFingerprint( + tenantId: string, + repositoryId: string, + commitHash: string, + rulePackVersion: string, + evaluations: RuleRegressionEvaluation[], +): string { + const summary = evaluations + .map((e) => `${e.ruleId}:${e.status}:${e.breaches.length}`) + .sort() + .join(","); + const raw = `${tenantId}:${repositoryId}:${commitHash}:${rulePackVersion}:${summary}`; + + // Deterministic fast hash (djb2) + let hash = 5381; + for (let i = 0; i < raw.length; i++) { + hash = ((hash << 5) + hash) ^ raw.charCodeAt(i); + } + return (hash >>> 0).toString(16).padStart(8, "0"); +} diff --git a/apps/api/src/rules/performance-regression/rule-performance-metrics.spec.ts b/apps/api/src/rules/performance-regression/rule-performance-metrics.spec.ts new file mode 100644 index 00000000..7e2d1c5c --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-metrics.spec.ts @@ -0,0 +1,76 @@ +import { + ConsoleRulePerformanceLogger, + NoopRulePerformanceMetrics, + RulePerformanceMetrics, + sanitizeLogFields, +} from "./rule-performance-metrics"; + +describe("RulePerformanceMetrics and Logging", () => { + describe("sanitizeLogFields", () => { + it("strips sensitive tokens, keys, passwords, and file paths", () => { + const sensitive = { + tenantId: "tenant-1", + status: "breached", + apiToken: "secret-token-123", + privateKey: "0xabcdef", + password: "supersecretpassword", + filePath: "/usr/src/secret/contract.sol", + sourceCode: "contract Foo {}", + nested: { + authToken: "bearer 123", + ruleId: "soroban-storage-rent", + }, + }; + + const sanitized = sanitizeLogFields(sensitive); + expect(sanitized.tenantId).toBe("tenant-1"); + expect(sanitized.status).toBe("breached"); + expect(sanitized.apiToken).toBeUndefined(); + expect(sanitized.privateKey).toBeUndefined(); + expect(sanitized.password).toBeUndefined(); + expect(sanitized.filePath).toBeUndefined(); + expect(sanitized.sourceCode).toBeUndefined(); + + const nested = sanitized.nested as Record; + expect(nested.authToken).toBeUndefined(); + expect(nested.ruleId).toBe("soroban-storage-rent"); + }); + + it("handles Error instances safely", () => { + const fields = { + error: new Error("Validation failure"), + }; + const sanitized = sanitizeLogFields(fields); + expect(sanitized.error).toBe("Validation failure"); + }); + }); + + describe("ConsoleRulePerformanceLogger", () => { + it("logs without throwing", () => { + const logger = new ConsoleRulePerformanceLogger("debug"); + expect(() => { + logger.debug("test_debug", { count: 1 }); + logger.info("test_info", { tenantId: "tenant-1" }); + logger.warn("test_warn", { warning: true }); + logger.error("test_error", { error: "fatal" }); + }).not.toThrow(); + }); + }); + + describe("RulePerformanceMetrics", () => { + it("allows recording metrics via NoopRulePerformanceMetrics", () => { + const metrics: RulePerformanceMetrics = new NoopRulePerformanceMetrics(); + expect(() => { + metrics.incrementEvaluations({ status: "passed" }); + metrics.incrementBreaches({ + ruleId: "soroban-storage-rent", + metric: "duration_ms", + severity: "breach", + }); + metrics.incrementErrors({ code: "STORE_UNAVAILABLE" }); + metrics.observeEvaluationDuration(0.045); + metrics.observeRuleDuration({ ruleId: "soroban-storage-rent" }, 0.012); + }).not.toThrow(); + }); + }); +}); diff --git a/apps/api/src/rules/performance-regression/rule-performance-metrics.ts b/apps/api/src/rules/performance-regression/rule-performance-metrics.ts new file mode 100644 index 00000000..a1ebd69d --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-metrics.ts @@ -0,0 +1,123 @@ +/** + * Metrics and structured logging for rule performance regression tracking (#1251). + * + * All metric labels are strictly bounded (ruleId, status, metric kind). + * All structured logs are strictly sanitized to never include repository paths, + * file contents, tokens, or raw exception strings. + */ + +import { + PerformanceRegressionStatus, + RegressionMetricKind, + RulePerformanceErrorCode, +} from "./rule-performance-types"; + +export interface RulePerformanceMetrics { + incrementEvaluations(labels: { status: PerformanceRegressionStatus }): void; + incrementBreaches(labels: { + ruleId: string; + metric: RegressionMetricKind; + severity: "warning" | "breach"; + }): void; + incrementErrors(labels: { code: RulePerformanceErrorCode }): void; + observeEvaluationDuration(seconds: number): void; + observeRuleDuration(labels: { ruleId: string }, seconds: number): void; +} + +export class NoopRulePerformanceMetrics implements RulePerformanceMetrics { + incrementEvaluations(): void {} + incrementBreaches(): void {} + incrementErrors(): void {} + observeEvaluationDuration(): void {} + observeRuleDuration(): void {} +} + +export type LogLevel = "debug" | "info" | "warn" | "error"; + +export interface LogEntry { + level: LogLevel; + event: string; + fields: Record; + timestamp: string; +} + +export interface RulePerformanceLogger { + debug(event: string, fields?: Record): void; + info(event: string, fields?: Record): void; + warn(event: string, fields?: Record): void; + error(event: string, fields?: Record): void; +} + +const DISALLOWED_KEY_PATTERNS = [ + /token/i, + /key/i, + /secret/i, + /auth/i, + /password/i, + /credential/i, + /path/i, + /file/i, + /code/i, + /source/i, +]; + +export function sanitizeLogFields( + fields: Record = {}, +): Record { + const sanitized: Record = {}; + for (const [k, v] of Object.entries(fields)) { + if (DISALLOWED_KEY_PATTERNS.some((pattern) => pattern.test(k))) { + continue; + } + if (v instanceof Error) { + sanitized[k] = v.message; + } else if (typeof v === "object" && v !== null && !Array.isArray(v)) { + sanitized[k] = sanitizeLogFields(v as Record); + } else { + sanitized[k] = v; + } + } + return sanitized; +} + +export class ConsoleRulePerformanceLogger implements RulePerformanceLogger { + constructor(private readonly minLevel: LogLevel = "info") {} + + private shouldLog(level: LogLevel): boolean { + const levels: LogLevel[] = ["debug", "info", "warn", "error"]; + return levels.indexOf(level) >= levels.indexOf(this.minLevel); + } + + private log(level: LogLevel, event: string, fields?: Record): void { + if (!this.shouldLog(level)) return; + const sanitized = sanitizeLogFields(fields); + const entry: LogEntry = { + level, + event, + fields: sanitized, + timestamp: new Date().toISOString(), + }; + const line = JSON.stringify(entry); + if (level === "error") { + process.stderr.write(line + "\n"); + } else { + process.stdout.write(line + "\n"); + } + } + + debug(event: string, fields?: Record): void { + this.log("debug", event, fields); + } + + info(event: string, fields?: Record): void { + this.log("info", event, fields); + } + + warn(event: string, fields?: Record): void { + this.log("warn", event, fields); + } + + error(event: string, fields?: Record): void { + this.log("error", event, fields); + } +} diff --git a/apps/api/src/rules/performance-regression/rule-performance-store.spec.ts b/apps/api/src/rules/performance-regression/rule-performance-store.spec.ts new file mode 100644 index 00000000..3684db2f --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-store.spec.ts @@ -0,0 +1,152 @@ +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; +import { + FileRulePerformanceStore, + InMemoryRulePerformanceStore, + RulePerformanceStore, +} from "./rule-performance-store"; +import { + EvaluationSummary, + RegressionTrackingScanPin, + RulePerformanceBaseline, +} from "./rule-performance-types"; + +describe("RulePerformanceStore", () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "gasguard-perf-store-test-")); + }); + + afterEach(() => { + if (fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + const testStores: Array<{ name: string; create: () => RulePerformanceStore }> = [ + { name: "InMemoryRulePerformanceStore", create: () => new InMemoryRulePerformanceStore() }, + { name: "FileRulePerformanceStore", create: () => new FileRulePerformanceStore(tmpDir) }, + ]; + + for (const { name, create } of testStores) { + describe(name, () => { + let store: RulePerformanceStore; + + beforeEach(() => { + store = create(); + }); + + const sampleBaseline: RulePerformanceBaseline = { + tenantId: "tenant-a", + repositoryId: "soroban-contracts", + commitHash: "commit-101", + rulePackVersion: "1.2.0", + ruleId: "soroban-storage-rent", + sampleCount: 5, + meanDurationMs: 120, + p95DurationMs: 150, + meanMemoryBytes: 500000, + p95MemoryBytes: 600000, + errorRate: 0, + createdAt: 1000, + updatedAt: 1000, + }; + + it("saves and retrieves baselines by commit and latest", async () => { + await store.saveBaseline(sampleBaseline); + + const byCommit = await store.getBaseline( + "tenant-a", + "soroban-contracts", + "soroban-storage-rent", + "commit-101", + ); + expect(byCommit).toBeDefined(); + expect(byCommit?.meanDurationMs).toBe(120); + + const latest = await store.getBaseline( + "tenant-a", + "soroban-contracts", + "soroban-storage-rent", + ); + expect(latest).toBeDefined(); + expect(latest?.ruleId).toBe("soroban-storage-rent"); + }); + + it("enforces tenant isolation", async () => { + await store.saveBaseline(sampleBaseline); + + const otherTenant = await store.getBaseline( + "tenant-b", + "soroban-contracts", + "soroban-storage-rent", + ); + expect(otherTenant).toBeUndefined(); + }); + + it("lists all baselines for a repository", async () => { + await store.saveBaseline(sampleBaseline); + await store.saveBaseline({ + ...sampleBaseline, + ruleId: "soroban-cpu-cost", + meanDurationMs: 80, + }); + + const all = await store.getBaselinesForRepository("tenant-a", "soroban-contracts"); + expect(all.size).toBe(2); + expect(all.get("soroban-storage-rent")?.meanDurationMs).toBe(120); + expect(all.get("soroban-cpu-cost")?.meanDurationMs).toBe(80); + }); + + it("persists scan pins with write-once semantics", async () => { + const pin: RegressionTrackingScanPin = { + tenantId: "tenant-a", + scanId: "scan-999", + repositoryId: "soroban-contracts", + commitHash: "commit-101", + rulePackVersion: "1.2.0", + pinnedAt: 2000, + fingerprint: "fp1234", + }; + + const firstSave = await store.saveScanPin(pin); + expect(firstSave.scanId).toBe("scan-999"); + + // Attempting to overwrite with different fingerprint returns original pin + const secondSave = await store.saveScanPin({ + ...pin, + fingerprint: "fp-modified", + }); + expect(secondSave.fingerprint).toBe("fp1234"); + + const retrieved = await store.getScanPin("tenant-a", "scan-999"); + expect(retrieved?.fingerprint).toBe("fp1234"); + }); + + it("saves and retrieves evaluation summaries", async () => { + const evalSummary: EvaluationSummary = { + tenantId: "tenant-a", + repositoryId: "soroban-contracts", + commitHash: "commit-101", + rulePackVersion: "1.2.0", + status: "breached", + totalRulesEvaluated: 1, + passedCount: 0, + warningCount: 0, + breachedCount: 1, + skippedRulesCount: 0, + ruleEvaluations: [], + evaluatedAt: 3000, + fingerprint: "eval-fp", + }; + + await store.saveEvaluation(evalSummary); + const retrieved = await store.getLatestEvaluation("tenant-a", "soroban-contracts"); + expect(retrieved).toBeDefined(); + expect(retrieved?.status).toBe("breached"); + }); + }); + } +}); diff --git a/apps/api/src/rules/performance-regression/rule-performance-store.ts b/apps/api/src/rules/performance-regression/rule-performance-store.ts new file mode 100644 index 00000000..f7996ef9 --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-store.ts @@ -0,0 +1,320 @@ +/** + * Persistence layer for rule performance regression baselines and evaluations (#1251). + * + * Provides tenant-isolated storage boundaries with both in-memory and file-based + * implementations. Enforces optimistic concurrency / write-once semantics for scan pins, + * and maintains per-tenant, per-repository baseline records. + */ + +import * as fs from "fs"; +import * as path from "path"; +import { + EvaluationSummary, + RegressionTrackingScanPin, + RulePerformanceBaseline, + RulePerformanceRegressionError, +} from "./rule-performance-types"; + +export interface RulePerformanceStore { + saveBaseline(baseline: RulePerformanceBaseline): Promise; + getBaseline( + tenantId: string, + repositoryId: string, + ruleId: string, + commitHash?: string, + ): Promise; + getBaselinesForRepository( + tenantId: string, + repositoryId: string, + commitHash?: string, + ): Promise>; + saveEvaluation(evaluation: EvaluationSummary): Promise; + getLatestEvaluation( + tenantId: string, + repositoryId: string, + ): Promise; + saveScanPin(pin: RegressionTrackingScanPin): Promise; + getScanPin( + tenantId: string, + scanId: string, + ): Promise; +} + +export class InMemoryRulePerformanceStore implements RulePerformanceStore { + // Key: `${tenantId}:${repositoryId}:${ruleId}:${commitHash}` + private readonly baselines = new Map(); + // Key: `${tenantId}:${repositoryId}:latest` + private readonly latestBaselinesByRepo = new Map>(); + // Key: `${tenantId}:${repositoryId}` + private readonly evaluations = new Map(); + // Key: `${tenantId}:${scanId}` + private readonly pins = new Map(); + + async saveBaseline(baseline: RulePerformanceBaseline): Promise { + const key = `${baseline.tenantId}:${baseline.repositoryId}:${baseline.ruleId}:${baseline.commitHash}`; + this.baselines.set(key, { ...baseline }); + + const repoKey = `${baseline.tenantId}:${baseline.repositoryId}`; + if (!this.latestBaselinesByRepo.has(repoKey)) { + this.latestBaselinesByRepo.set(repoKey, new Map()); + } + this.latestBaselinesByRepo.get(repoKey)!.set(baseline.ruleId, { ...baseline }); + } + + async getBaseline( + tenantId: string, + repositoryId: string, + ruleId: string, + commitHash?: string, + ): Promise { + if (commitHash) { + const key = `${tenantId}:${repositoryId}:${ruleId}:${commitHash}`; + return this.baselines.get(key); + } + const repoKey = `${tenantId}:${repositoryId}`; + return this.latestBaselinesByRepo.get(repoKey)?.get(ruleId); + } + + async getBaselinesForRepository( + tenantId: string, + repositoryId: string, + commitHash?: string, + ): Promise> { + const result = new Map(); + if (commitHash) { + const prefix = `${tenantId}:${repositoryId}:`; + const suffix = `:${commitHash}`; + for (const [k, v] of this.baselines.entries()) { + if (k.startsWith(prefix) && k.endsWith(suffix)) { + result.set(v.ruleId, { ...v }); + } + } + return result; + } + + const repoKey = `${tenantId}:${repositoryId}`; + const latestMap = this.latestBaselinesByRepo.get(repoKey); + if (latestMap) { + for (const [ruleId, baseline] of latestMap.entries()) { + result.set(ruleId, { ...baseline }); + } + } + return result; + } + + async saveEvaluation(evaluation: EvaluationSummary): Promise { + const key = `${evaluation.tenantId}:${evaluation.repositoryId}`; + this.evaluations.set(key, { ...evaluation }); + } + + async getLatestEvaluation( + tenantId: string, + repositoryId: string, + ): Promise { + const key = `${tenantId}:${repositoryId}`; + return this.evaluations.get(key); + } + + async saveScanPin(pin: RegressionTrackingScanPin): Promise { + const key = `${pin.tenantId}:${pin.scanId}`; + const existing = this.pins.get(key); + if (existing) { + return existing; + } + this.pins.set(key, { ...pin }); + return pin; + } + + async getScanPin( + tenantId: string, + scanId: string, + ): Promise { + const key = `${tenantId}:${scanId}`; + return this.pins.get(key); + } +} + +export class FileRulePerformanceStore implements RulePerformanceStore { + constructor(private readonly baseDir: string) { + if (!fs.existsSync(baseDir)) { + fs.mkdirSync(baseDir, { recursive: true }); + } + } + + private sanitizeFilename(id: string): string { + return id.replace(/[^a-zA-Z0-9_-]/g, "_"); + } + + private getTenantDir(tenantId: string): string { + const dir = path.join(this.baseDir, this.sanitizeFilename(tenantId)); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + return dir; + } + + async saveBaseline(baseline: RulePerformanceBaseline): Promise { + try { + const tenantDir = this.getTenantDir(baseline.tenantId); + const file = path.join( + tenantDir, + `baseline_${this.sanitizeFilename(baseline.repositoryId)}_${this.sanitizeFilename(baseline.ruleId)}_${baseline.commitHash}.json`, + ); + await fs.promises.writeFile(file, JSON.stringify(baseline, null, 2), "utf8"); + + // Also update latest pointer + const latestFile = path.join( + tenantDir, + `baseline_latest_${this.sanitizeFilename(baseline.repositoryId)}_${this.sanitizeFilename(baseline.ruleId)}.json`, + ); + await fs.promises.writeFile(latestFile, JSON.stringify(baseline, null, 2), "utf8"); + } catch (err) { + throw new RulePerformanceRegressionError( + "STORE_UNAVAILABLE", + "Failed to write baseline to disk", + err, + ); + } + } + + async getBaseline( + tenantId: string, + repositoryId: string, + ruleId: string, + commitHash?: string, + ): Promise { + try { + const tenantDir = this.getTenantDir(tenantId); + const file = commitHash + ? path.join( + tenantDir, + `baseline_${this.sanitizeFilename(repositoryId)}_${this.sanitizeFilename(ruleId)}_${commitHash}.json`, + ) + : path.join( + tenantDir, + `baseline_latest_${this.sanitizeFilename(repositoryId)}_${this.sanitizeFilename(ruleId)}.json`, + ); + + if (!fs.existsSync(file)) return undefined; + const raw = await fs.promises.readFile(file, "utf8"); + return JSON.parse(raw); + } catch (err) { + throw new RulePerformanceRegressionError( + "STORE_UNAVAILABLE", + "Failed to read baseline from disk", + err, + ); + } + } + + async getBaselinesForRepository( + tenantId: string, + repositoryId: string, + commitHash?: string, + ): Promise> { + try { + const result = new Map(); + const tenantDir = this.getTenantDir(tenantId); + const files = await fs.promises.readdir(tenantDir); + + const targetPrefix = commitHash + ? `baseline_${this.sanitizeFilename(repositoryId)}_` + : `baseline_latest_${this.sanitizeFilename(repositoryId)}_`; + + for (const f of files) { + if (f.startsWith(targetPrefix) && f.endsWith(".json")) { + if (commitHash && !f.endsWith(`_${commitHash}.json`)) { + continue; + } + const raw = await fs.promises.readFile(path.join(tenantDir, f), "utf8"); + const baseline: RulePerformanceBaseline = JSON.parse(raw); + result.set(baseline.ruleId, baseline); + } + } + return result; + } catch (err) { + throw new RulePerformanceRegressionError( + "STORE_UNAVAILABLE", + "Failed to read baselines for repository from disk", + err, + ); + } + } + + async saveEvaluation(evaluation: EvaluationSummary): Promise { + try { + const tenantDir = this.getTenantDir(evaluation.tenantId); + const file = path.join( + tenantDir, + `eval_${this.sanitizeFilename(evaluation.repositoryId)}.json`, + ); + await fs.promises.writeFile(file, JSON.stringify(evaluation, null, 2), "utf8"); + } catch (err) { + throw new RulePerformanceRegressionError( + "STORE_UNAVAILABLE", + "Failed to save evaluation to disk", + err, + ); + } + } + + async getLatestEvaluation( + tenantId: string, + repositoryId: string, + ): Promise { + try { + const tenantDir = this.getTenantDir(tenantId); + const file = path.join( + tenantDir, + `eval_${this.sanitizeFilename(repositoryId)}.json`, + ); + if (!fs.existsSync(file)) return undefined; + const raw = await fs.promises.readFile(file, "utf8"); + return JSON.parse(raw); + } catch (err) { + throw new RulePerformanceRegressionError( + "STORE_UNAVAILABLE", + "Failed to read latest evaluation from disk", + err, + ); + } + } + + async saveScanPin(pin: RegressionTrackingScanPin): Promise { + try { + const tenantDir = this.getTenantDir(pin.tenantId); + const file = path.join(tenantDir, `pin_${this.sanitizeFilename(pin.scanId)}.json`); + if (fs.existsSync(file)) { + const raw = await fs.promises.readFile(file, "utf8"); + return JSON.parse(raw); + } + await fs.promises.writeFile(file, JSON.stringify(pin, null, 2), "utf8"); + return pin; + } catch (err) { + throw new RulePerformanceRegressionError( + "STORE_UNAVAILABLE", + "Failed to save scan pin to disk", + err, + ); + } + } + + async getScanPin( + tenantId: string, + scanId: string, + ): Promise { + try { + const tenantDir = this.getTenantDir(tenantId); + const file = path.join(tenantDir, `pin_${this.sanitizeFilename(scanId)}.json`); + if (!fs.existsSync(file)) return undefined; + const raw = await fs.promises.readFile(file, "utf8"); + return JSON.parse(raw); + } catch (err) { + throw new RulePerformanceRegressionError( + "STORE_UNAVAILABLE", + "Failed to read scan pin from disk", + err, + ); + } + } +} diff --git a/apps/api/src/rules/performance-regression/rule-performance-types.ts b/apps/api/src/rules/performance-regression/rule-performance-types.ts new file mode 100644 index 00000000..09307df4 --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-types.ts @@ -0,0 +1,133 @@ +/** + * Shared types for rule performance regression thresholds tracking (#1251). + * + * Tracks execution duration, memory consumption, and error rates of rules across + * repository scans and rule-pack versions. Flags performance regressions against + * configured baseline thresholds, integrates with quality gates, and supports + * tenant isolation, partial failure handling, and deterministic evaluation. + */ + +export const PERFORMANCE_STATUSES = [ + "passed", + "warning", + "breached", +] as const; + +export type PerformanceRegressionStatus = (typeof PERFORMANCE_STATUSES)[number]; + +export const REGRESSION_METRIC_KINDS = [ + "duration_ms", + "memory_bytes", + "error_rate", +] as const; + +export type RegressionMetricKind = (typeof REGRESSION_METRIC_KINDS)[number]; + +export interface RulePerformanceThresholds { + /** Maximum allowable absolute duration in ms before breach. */ + maxDurationMs?: number; + /** Maximum allowable percentage increase in duration compared to baseline (e.g. 50 = +50%). */ + maxDurationDeltaPercent?: number; + /** Maximum allowable absolute memory in bytes before breach. */ + maxMemoryBytes?: number; + /** Maximum allowable percentage increase in memory compared to baseline. */ + maxMemoryDeltaPercent?: number; + /** Maximum allowable error/timeout rate (0.0 - 1.0). */ + maxErrorRate?: number; + /** Warning percentage factor (e.g. 0.8 means warning fires at 80% of threshold). Default: 0.8. */ + warningFactor?: number; +} + +export interface RuleExecutionSample { + ruleId: string; + packId?: string; + packVersion?: string; + durationMs: number; + memoryBytes?: number; + status: "ok" | "error" | "timeout"; + findingsCount: number; +} + +export interface RulePerformanceBaseline { + tenantId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + ruleId: string; + sampleCount: number; + meanDurationMs: number; + p95DurationMs: number; + meanMemoryBytes: number; + p95MemoryBytes: number; + errorRate: number; + createdAt: number; + updatedAt: number; +} + +export interface RuleMetricBreach { + metric: RegressionMetricKind; + severity: "warning" | "breach"; + currentValue: number; + baselineValue?: number; + thresholdValue: number; + deltaPercent?: number; + message: string; +} + +export interface RuleRegressionEvaluation { + ruleId: string; + status: PerformanceRegressionStatus; + breaches: RuleMetricBreach[]; + currentDurationMs: number; + baselineDurationMs?: number; + currentMemoryBytes?: number; + baselineMemoryBytes?: number; + errorRate: number; +} + +export interface EvaluationSummary { + tenantId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + status: PerformanceRegressionStatus; + totalRulesEvaluated: number; + passedCount: number; + warningCount: number; + breachedCount: number; + skippedRulesCount: number; + ruleEvaluations: RuleRegressionEvaluation[]; + evaluatedAt: number; + fingerprint: string; +} + +export interface RegressionTrackingScanPin { + tenantId: string; + scanId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + pinnedAt: number; + baselineCommitHash?: string; + fingerprint: string; +} + +export type RulePerformanceErrorCode = + | "TRACKING_DISABLED" + | "INVALID_INPUT" + | "INVALID_CONFIG" + | "STORE_UNAVAILABLE" + | "STORE_CONFLICT" + | "UNSUPPORTED_REPOSITORY" + | "BASELINE_NOT_FOUND"; + +export class RulePerformanceRegressionError extends Error { + constructor( + public readonly code: RulePerformanceErrorCode, + message: string, + public readonly details?: unknown, + ) { + super(`[${code}] ${message}`); + this.name = "RulePerformanceRegressionError"; + } +} diff --git a/apps/api/src/rules/performance-regression/rule-performance-worker.ts b/apps/api/src/rules/performance-regression/rule-performance-worker.ts new file mode 100644 index 00000000..e3914f52 --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance-worker.ts @@ -0,0 +1,103 @@ +/** + * Worker-side adapter for rule performance regression tracking (#1251). + * + * Integrates directly with scan jobs, wrapping rule execution or analyzer runs. + * Gathers timing and memory metrics safely, passes them to RulePerformanceService, + * and handles partial failures and exceptions gracefully. + */ + +import { + RulePerformanceService, + TrackScanExecutionInput, +} from "./rule-performance.service"; +import { + EvaluationSummary, + RuleExecutionSample, +} from "./rule-performance-types"; + +export interface ScanWorkerRuleExecution { + ruleId: string; + packId?: string; + packVersion?: string; + execute: () => Promise | unknown[]; +} + +export interface WorkerScanRuleExecutionOptions { + service: RulePerformanceService; + tenantId: string; + scanId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + baselineCommitHash?: string; + failOnBreach?: boolean; +} + +export async function runRulesWithPerformanceTracking( + rules: ScanWorkerRuleExecution[], + options: WorkerScanRuleExecutionOptions, +): Promise<{ + results: Map; + evaluation: EvaluationSummary; +}> { + const { + service, + tenantId, + scanId, + repositoryId, + commitHash, + rulePackVersion, + baselineCommitHash, + failOnBreach = false, + } = options; + + const results = new Map(); + const samples: RuleExecutionSample[] = []; + + for (const rule of rules) { + const startMemory = process.memoryUsage ? process.memoryUsage().heapUsed : 0; + const startTs = Date.now(); + let findings: unknown[] = []; + let status: "ok" | "error" | "timeout" = "ok"; + + try { + findings = await rule.execute(); + results.set(rule.ruleId, findings); + } catch (err) { + status = "error"; + results.set(rule.ruleId, []); + } + + const durationMs = Date.now() - startTs; + const endMemory = process.memoryUsage ? process.memoryUsage().heapUsed : 0; + const memoryBytes = Math.max(0, endMemory - startMemory); + + samples.push({ + ruleId: rule.ruleId, + packId: rule.packId, + packVersion: rule.packVersion, + durationMs, + memoryBytes, + status, + findingsCount: findings.length, + }); + } + + const evaluation = await service.evaluateScanPerformance({ + tenantId, + scanId, + repositoryId, + commitHash, + rulePackVersion, + samples, + baselineCommitHash, + }); + + if (failOnBreach && evaluation.status === "breached") { + throw new Error( + `Scan failed rule performance regression quality gate: ${evaluation.breachedCount} rule(s) breached thresholds.`, + ); + } + + return { results, evaluation }; +} diff --git a/apps/api/src/rules/performance-regression/rule-performance.integration.spec.ts b/apps/api/src/rules/performance-regression/rule-performance.integration.spec.ts new file mode 100644 index 00000000..9738f03f --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance.integration.spec.ts @@ -0,0 +1,131 @@ +import { RulePerformanceService } from "./rule-performance.service"; +import { InMemoryRulePerformanceStore } from "./rule-performance-store"; +import { + runRulesWithPerformanceTracking, + ScanWorkerRuleExecution, +} from "./rule-performance-worker"; +import realisticFixture from "./__fixtures__/realistic-repo-fixture.json"; +import regressionFixture from "./__fixtures__/regression-case-fixture.json"; + +describe("RulePerformanceIntegration", () => { + let service: RulePerformanceService; + let store: InMemoryRulePerformanceStore; + + beforeEach(() => { + store = new InMemoryRulePerformanceStore(); + service = new RulePerformanceService({ + store, + config: { + defaultThresholds: { + maxDurationMs: 2000, + maxDurationDeltaPercent: 50, + maxMemoryBytes: 50 * 1024 * 1024, + maxMemoryDeltaPercent: 50, + }, + }, + }); + }); + + it("handles realistic repository scan flow and establishes baseline", async () => { + // 1. Establish baseline from realistic fixture + const count = await service.establishBaselines({ + tenantId: realisticFixture.tenantId, + repositoryId: realisticFixture.repositoryId, + commitHash: realisticFixture.baselineCommit, + rulePackVersion: realisticFixture.rulePackVersion, + samples: realisticFixture.baselineSamples as any, + }); + expect(count).toBe(3); + + // 2. Worker executes scan with realistic rules + const rules: ScanWorkerRuleExecution[] = realisticFixture.normalScanSamples.map( + (s) => ({ + ruleId: s.ruleId, + execute: async () => [{ finding: `finding-for-${s.ruleId}` }], + }), + ); + + const { results, evaluation } = await runRulesWithPerformanceTracking( + rules, + { + service, + tenantId: realisticFixture.tenantId, + scanId: "scan-normal-001", + repositoryId: realisticFixture.repositoryId, + commitHash: realisticFixture.currentCommit, + rulePackVersion: realisticFixture.rulePackVersion, + }, + ); + + expect(results.size).toBe(3); + expect(evaluation.status).toBe("passed"); + expect(evaluation.breachedCount).toBe(0); + }); + + it("evaluates regression case fixture and flags breached rules", async () => { + // Establish baseline + await service.establishBaselines({ + tenantId: regressionFixture.tenantId, + repositoryId: regressionFixture.repositoryId, + commitHash: regressionFixture.baselineCommit, + rulePackVersion: regressionFixture.rulePackVersion, + samples: realisticFixture.baselineSamples as any, + }); + + // Evaluate regressed samples + const evalSummary = await service.evaluateScanPerformance({ + tenantId: regressionFixture.tenantId, + scanId: "scan-regressed-001", + repositoryId: regressionFixture.repositoryId, + commitHash: regressionFixture.regressedCommit, + rulePackVersion: regressionFixture.rulePackVersion, + samples: regressionFixture.regressedSamples as any, + }); + + expect(evalSummary.status).toBe("breached"); + expect(evalSummary.breachedCount).toBeGreaterThanOrEqual(1); + + const breachedRuleIds = evalSummary.ruleEvaluations + .filter((e) => e.status === "breached") + .map((e) => e.ruleId); + + expect(breachedRuleIds).toEqual( + expect.arrayContaining(regressionFixture.expectedBreachedRules), + ); + }); + + it("handles worker partial failure gracefully without dropping other rule outcomes", async () => { + const rules: ScanWorkerRuleExecution[] = [ + { + ruleId: "soroban-constant-propagation", + execute: () => [{ id: "const-1" }], + }, + { + ruleId: "broken-rule", + execute: () => { + throw new Error("Syntax error in AST parsing"); + }, + }, + ]; + + const { results, evaluation } = await runRulesWithPerformanceTracking( + rules, + { + service, + tenantId: "tenant-fault-tolerance", + scanId: "scan-partial-001", + repositoryId: "repo-demo", + commitHash: "c-fault", + rulePackVersion: "1.0.0", + }, + ); + + expect(results.get("soroban-constant-propagation")).toEqual([{ id: "const-1" }]); + expect(results.get("broken-rule")).toEqual([]); + expect(evaluation.totalRulesEvaluated).toBe(2); + + const brokenEval = evaluation.ruleEvaluations.find((e) => e.ruleId === "broken-rule"); + expect(brokenEval).toBeDefined(); + expect(brokenEval?.errorRate).toBe(1.0); + }); +}); diff --git a/apps/api/src/rules/performance-regression/rule-performance.service.spec.ts b/apps/api/src/rules/performance-regression/rule-performance.service.spec.ts new file mode 100644 index 00000000..9d72cb2d --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance.service.spec.ts @@ -0,0 +1,147 @@ +import { RulePerformanceService } from "./rule-performance.service"; +import { InMemoryRulePerformanceStore } from "./rule-performance-store"; +import { RuleExecutionSample } from "./rule-performance-types"; + +describe("RulePerformanceService", () => { + let service: RulePerformanceService; + let store: InMemoryRulePerformanceStore; + + beforeEach(() => { + store = new InMemoryRulePerformanceStore(); + service = new RulePerformanceService({ + store, + config: { + defaultThresholds: { + maxDurationMs: 1000, + maxDurationDeltaPercent: 50, + maxMemoryBytes: 2000000, + maxMemoryDeltaPercent: 50, + }, + }, + clock: () => 100000, + }); + }); + + it("pins scan attempts and reuses the pin across retries", async () => { + const pinRes1 = await service.pinScan({ + tenantId: "tenant-alpha", + scanId: "scan-123", + repositoryId: "repo-xyz", + commitHash: "c1", + rulePackVersion: "1.0.0", + }); + expect(pinRes1.status).toBe("created"); + expect(pinRes1.pin.scanId).toBe("scan-123"); + + const pinRes2 = await service.pinScan({ + tenantId: "tenant-alpha", + scanId: "scan-123", + repositoryId: "repo-xyz", + commitHash: "c1", + rulePackVersion: "1.0.0", + }); + expect(pinRes2.status).toBe("reused"); + expect(pinRes2.pin.fingerprint).toBe(pinRes1.pin.fingerprint); + }); + + it("establishes baselines and detects performance regressions in scans", async () => { + // 1. Establish baseline + const baselineSamples: RuleExecutionSample[] = [ + { + ruleId: "soroban-storage-rent", + durationMs: 200, + memoryBytes: 500000, + status: "ok", + findingsCount: 1, + }, + ]; + + const established = await service.establishBaselines({ + tenantId: "tenant-alpha", + repositoryId: "repo-xyz", + commitHash: "c1", + rulePackVersion: "1.0.0", + samples: baselineSamples, + }); + expect(established).toBe(1); + + // 2. Run scan with normal metrics -> passes + const normalSamples: RuleExecutionSample[] = [ + { + ruleId: "soroban-storage-rent", + durationMs: 210, + memoryBytes: 510000, + status: "ok", + findingsCount: 1, + }, + ]; + + const eval1 = await service.evaluateScanPerformance({ + tenantId: "tenant-alpha", + scanId: "scan-101", + repositoryId: "repo-xyz", + commitHash: "c2", + rulePackVersion: "1.0.0", + samples: normalSamples, + }); + expect(eval1.status).toBe("passed"); + expect(eval1.breachedCount).toBe(0); + + // 3. Run scan with regressed duration (+150%) -> breaches + const regressedSamples: RuleExecutionSample[] = [ + { + ruleId: "soroban-storage-rent", + durationMs: 500, // 500 vs 200 is +150% regression + memoryBytes: 520000, + status: "ok", + findingsCount: 1, + }, + ]; + + const eval2 = await service.evaluateScanPerformance({ + tenantId: "tenant-alpha", + scanId: "scan-102", + repositoryId: "repo-xyz", + commitHash: "c3", + rulePackVersion: "1.0.0", + samples: regressedSamples, + }); + expect(eval2.status).toBe("breached"); + expect(eval2.breachedCount).toBe(1); + expect(eval2.ruleEvaluations[0].breaches[0].deltaPercent).toBe(150); + }); + + it("handles kill-switch being disabled gracefully", async () => { + const disabledService = new RulePerformanceService({ + store, + config: { enabled: false }, + }); + + const evalRes = await disabledService.evaluateScanPerformance({ + tenantId: "tenant-alpha", + scanId: "scan-disabled", + repositoryId: "repo-xyz", + commitHash: "c4", + rulePackVersion: "1.0.0", + samples: [ + { ruleId: "soroban-storage-rent", durationMs: 99999, status: "ok", findingsCount: 0 }, + ], + }); + + expect(evalRes.status).toBe("passed"); + expect(evalRes.totalRulesEvaluated).toBe(0); + }); + + it("validates tenant and repository identifiers", async () => { + await expect( + service.evaluateScanPerformance({ + tenantId: "", + scanId: "scan-1", + repositoryId: "repo-1", + commitHash: "c1", + rulePackVersion: "1.0.0", + samples: [], + }), + ).rejects.toThrow("[INVALID_INPUT]"); + }); +}); diff --git a/apps/api/src/rules/performance-regression/rule-performance.service.ts b/apps/api/src/rules/performance-regression/rule-performance.service.ts new file mode 100644 index 00000000..b16b7664 --- /dev/null +++ b/apps/api/src/rules/performance-regression/rule-performance.service.ts @@ -0,0 +1,311 @@ +/** + * Service for tracking rule performance regression thresholds (#1251). + * + * Integrates with analyzer execution, worker scan jobs, and reporting APIs. + * Supports tenant isolation, baseline creation, deterministic evaluation, + * and resilient error handling for partial failures and retries. + */ + +import { + DEFAULT_RULE_PERFORMANCE_CONFIG, + RulePerformanceConfig, + resolveRulePerformanceConfig, +} from "./rule-performance-config"; +import { + evaluateRulePerformance, + computeEvaluationFingerprint, +} from "./rule-performance-evaluator"; +import { + ConsoleRulePerformanceLogger, + NoopRulePerformanceMetrics, + RulePerformanceLogger, + RulePerformanceMetrics, +} from "./rule-performance-metrics"; +import { + InMemoryRulePerformanceStore, + RulePerformanceStore, +} from "./rule-performance-store"; +import { + EvaluationSummary, + RegressionTrackingScanPin, + RuleExecutionSample, + RulePerformanceBaseline, + RulePerformanceRegressionError, + RulePerformanceThresholds, +} from "./rule-performance-types"; + +export interface RulePerformanceServiceOptions { + store?: RulePerformanceStore; + config?: Partial; + logger?: RulePerformanceLogger; + metrics?: RulePerformanceMetrics; + clock?: () => number; +} + +export interface TrackScanExecutionInput { + tenantId: string; + scanId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + samples: RuleExecutionSample[]; + baselineCommitHash?: string; +} + +export interface SetRuleBaselineInput { + tenantId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + samples: RuleExecutionSample[]; +} + +export class RulePerformanceService { + private readonly store: RulePerformanceStore; + private readonly config: RulePerformanceConfig; + private readonly logger: RulePerformanceLogger; + private readonly metrics: RulePerformanceMetrics; + private readonly clock: () => number; + + constructor(options: RulePerformanceServiceOptions = {}) { + this.config = resolveRulePerformanceConfig(options.config || {}); + this.store = options.store || new InMemoryRulePerformanceStore(); + this.logger = options.logger || new ConsoleRulePerformanceLogger(); + this.metrics = options.metrics || new NoopRulePerformanceMetrics(); + this.clock = options.clock || (() => Date.now()); + } + + getConfig(): RulePerformanceConfig { + return this.config; + } + + /** + * Pins a scan attempt or retrieves an existing pinned scan. + * Ensures deterministic evaluation across scan retries. + */ + async pinScan(input: { + tenantId: string; + scanId: string; + repositoryId: string; + commitHash: string; + rulePackVersion: string; + baselineCommitHash?: string; + }): Promise<{ pin: RegressionTrackingScanPin; status: "created" | "reused" }> { + this.validateIdentifier(input.tenantId, "tenantId"); + this.validateIdentifier(input.scanId, "scanId"); + this.validateIdentifier(input.repositoryId, "repositoryId"); + + const existing = await this.store.getScanPin(input.tenantId, input.scanId); + if (existing) { + return { pin: existing, status: "reused" }; + } + + const fingerprint = computeEvaluationFingerprint( + input.tenantId, + input.repositoryId, + input.commitHash, + input.rulePackVersion, + [], + ); + + const newPin: RegressionTrackingScanPin = { + tenantId: input.tenantId, + scanId: input.scanId, + repositoryId: input.repositoryId, + commitHash: input.commitHash, + rulePackVersion: input.rulePackVersion, + baselineCommitHash: input.baselineCommitHash, + pinnedAt: this.clock(), + fingerprint, + }; + + const saved = await this.store.saveScanPin(newPin); + return { pin: saved, status: "created" }; + } + + /** + * Evaluate rule execution performance against configured thresholds and stored baselines. + */ + async evaluateScanPerformance( + input: TrackScanExecutionInput, + ): Promise { + if (!this.config.enabled) { + this.logger.info("Rule performance tracking is disabled via config"); + return { + tenantId: input.tenantId, + repositoryId: input.repositoryId, + commitHash: input.commitHash, + rulePackVersion: input.rulePackVersion, + status: "passed", + totalRulesEvaluated: 0, + passedCount: 0, + warningCount: 0, + breachedCount: 0, + skippedRulesCount: input.samples.length, + ruleEvaluations: [], + evaluatedAt: this.clock(), + fingerprint: "disabled", + }; + } + + this.validateIdentifier(input.tenantId, "tenantId"); + this.validateIdentifier(input.repositoryId, "repositoryId"); + this.validateIdentifier(input.scanId, "scanId"); + + const startTs = this.clock(); + + // Retrieve or establish scan pin for retries + const { pin } = await this.pinScan({ + tenantId: input.tenantId, + scanId: input.scanId, + repositoryId: input.repositoryId, + commitHash: input.commitHash, + rulePackVersion: input.rulePackVersion, + baselineCommitHash: input.baselineCommitHash, + }); + + const targetBaselineCommit = pin.baselineCommitHash || input.baselineCommitHash; + + // Load baselines + const baselines = await this.store.getBaselinesForRepository( + input.tenantId, + input.repositoryId, + targetBaselineCommit, + ); + + // Filter invalid or overflowing samples gracefully + const validSamples = this.filterAndValidateSamples(input.samples); + + const summary = evaluateRulePerformance({ + tenantId: input.tenantId, + repositoryId: input.repositoryId, + commitHash: input.commitHash, + rulePackVersion: input.rulePackVersion, + samples: validSamples, + baselines, + defaultThresholds: this.config.defaultThresholds, + ruleOverrides: this.config.ruleOverrides, + clock: this.clock, + }); + + await this.store.saveEvaluation(summary); + + // Record metrics + this.metrics.incrementEvaluations({ status: summary.status }); + for (const ruleEval of summary.ruleEvaluations) { + this.metrics.observeRuleDuration( + { ruleId: ruleEval.ruleId }, + ruleEval.currentDurationMs / 1000, + ); + for (const breach of ruleEval.breaches) { + this.metrics.incrementBreaches({ + ruleId: ruleEval.ruleId, + metric: breach.metric, + severity: breach.severity, + }); + } + } + this.metrics.observeEvaluationDuration((this.clock() - startTs) / 1000); + + this.logger.info("Rule performance evaluation completed", { + tenantId: input.tenantId, + repositoryId: input.repositoryId, + status: summary.status, + breachedCount: summary.breachedCount, + warningCount: summary.warningCount, + }); + + return summary; + } + + /** + * Save a set of execution samples as a new baseline for the given repo & commit. + */ + async establishBaselines(input: SetRuleBaselineInput): Promise { + this.validateIdentifier(input.tenantId, "tenantId"); + this.validateIdentifier(input.repositoryId, "repositoryId"); + + const samplesByRule = new Map(); + for (const s of input.samples) { + if (!samplesByRule.has(s.ruleId)) { + samplesByRule.set(s.ruleId, []); + } + samplesByRule.get(s.ruleId)!.push(s); + } + + let count = 0; + const now = this.clock(); + + for (const [ruleId, samples] of samplesByRule.entries()) { + const durations = samples.map((s) => s.durationMs).sort((a, b) => a - b); + const meanDuration = + durations.reduce((sum, d) => sum + d, 0) / durations.length; + const p95Duration = + durations[Math.min(Math.floor(durations.length * 0.95), durations.length - 1)]; + + const memories = samples + .map((s) => s.memoryBytes || 0) + .sort((a, b) => a - b); + const meanMemory = + memories.reduce((sum, m) => sum + m, 0) / memories.length; + const p95Memory = + memories[Math.min(Math.floor(memories.length * 0.95), memories.length - 1)]; + + const errors = samples.filter((s) => s.status !== "ok").length; + const errorRate = errors / samples.length; + + const baseline: RulePerformanceBaseline = { + tenantId: input.tenantId, + repositoryId: input.repositoryId, + commitHash: input.commitHash, + rulePackVersion: input.rulePackVersion, + ruleId, + sampleCount: samples.length, + meanDurationMs: Math.round(meanDuration * 100) / 100, + p95DurationMs: Math.round(p95Duration * 100) / 100, + meanMemoryBytes: Math.round(meanMemory), + p95MemoryBytes: Math.round(p95Memory), + errorRate: Math.round(errorRate * 1000) / 1000, + createdAt: now, + updatedAt: now, + }; + + await this.store.saveBaseline(baseline); + count++; + } + + this.logger.info("Rule performance baselines established", { + tenantId: input.tenantId, + repositoryId: input.repositoryId, + baselinesCount: count, + }); + + return count; + } + + private filterAndValidateSamples( + samples: RuleExecutionSample[], + ): RuleExecutionSample[] { + const valid: RuleExecutionSample[] = []; + for (const s of samples) { + if (!s.ruleId || s.ruleId.trim() === "") continue; + if (s.durationMs < 0 || !Number.isFinite(s.durationMs)) continue; + if (s.memoryBytes !== undefined && s.memoryBytes < 0) continue; + valid.push(s); + } + return valid.slice(0, this.config.maxSamplesPerRule * 100); + } + + private validateIdentifier(value: string, field: string): void { + if ( + typeof value !== "string" || + value.trim() === "" || + value.length > this.config.maxIdentifierLength + ) { + throw new RulePerformanceRegressionError( + "INVALID_INPUT", + `${field} must be a non-empty string of at most ${this.config.maxIdentifierLength} characters`, + ); + } + } +} diff --git a/docs/RULE_PERFORMANCE_REGRESSION_THRESHOLDS.md b/docs/RULE_PERFORMANCE_REGRESSION_THRESHOLDS.md new file mode 100644 index 00000000..a0478dd3 --- /dev/null +++ b/docs/RULE_PERFORMANCE_REGRESSION_THRESHOLDS.md @@ -0,0 +1,72 @@ +# Track Rule Performance Regression Thresholds + +Part of the **Quality and Evaluation** workstream (#1251). + +## Overview + +GasGuard executes static analysis and optimization rules against smart contracts and multi-language repositories. As rules evolve and rule packs update, regressions in execution time, memory footprint, or failure/timeout rates can degrade repository scan performance and impact CI/CD pipelines. + +The **Rule Performance Regression Thresholds** module provides automated, deterministic tracking of rule execution performance across repository scans and rule-pack versions. It detects metric degradation compared to established baselines, alerts operators through structured metrics, and enforces quality gates before regressions reach production. + +| Component | Location | +| --- | --- | +| Types and Error Codes | `apps/api/src/rules/performance-regression/rule-performance-types.ts` | +| Configuration (`RULE_PERFORMANCE_*`) | `apps/api/src/rules/performance-regression/rule-performance-config.ts` | +| Pure Deterministic Evaluator | `apps/api/src/rules/performance-regression/rule-performance-evaluator.ts` | +| Persistence (InMemory & File) | `apps/api/src/rules/performance-regression/rule-performance-store.ts` | +| Core Service Boundary | `apps/api/src/rules/performance-regression/rule-performance.service.ts` | +| Worker Adapter | `apps/api/src/rules/performance-regression/rule-performance-worker.ts` | +| Metrics and Structured Logs | `apps/api/src/rules/performance-regression/rule-performance-metrics.ts` | +| Fixtures (Realistic Repo & Regression) | `apps/api/src/rules/performance-regression/__fixtures__/` | + +--- + +## Architectural Principles & Design + +### 1. Determinism +- **Pure evaluation**: Given identical rule execution samples, baseline metrics, and configuration thresholds, the evaluator produces identical results (`passed`, `warning`, `breached`) and an identical djb2 fingerprint. +- **Scan Pinning**: Write-once scan pins store the exact evaluation parameters per `(tenantId, scanId)`. Retry attempts always reuse the established pin so transient worker restarts do not alter baseline comparison refs. + +### 2. Tenant Isolation +- Baselines, evaluations, and scan pins are strictly partitioned by `tenantId`. +- Both in-memory and disk persistence enforce isolated directory structures (`//...`), preventing data leakages across organizations. + +### 3. Metric Thresholds & Regression Math +Rules are evaluated against absolute ceilings as well as relative delta percentages against baseline records: +- **Duration (Execution time)**: + - Absolute threshold: `maxDurationMs` (default: 5,000ms) + - Relative regression: `maxDurationDeltaPercent` (default: +50% regression) +- **Memory Consumption**: + - Absolute threshold: `maxMemoryBytes` (default: 256MB) + - Relative regression: `maxMemoryDeltaPercent` (default: +50% regression) +- **Error / Timeout Rate**: + - Maximum ratio of non-ok executions: `maxErrorRate` (default: 0.05 / 5%) +- **Early Warnings**: + - Warning threshold factor: `warningFactor` (default: 0.8 / 80% of breach limit) + +--- + +## Configuration Reference + +Environment variables configure the default limits and behaviors: + +| Environment Variable | Default | Purpose | +| --- | --- | --- | +| `RULE_PERFORMANCE_TRACKING_ENABLED` | `true` | Master toggle and kill-switch | +| `RULE_PERFORMANCE_DEFAULT_MAX_DURATION_MS` | `5000` | Absolute duration limit in ms | +| `RULE_PERFORMANCE_DEFAULT_MAX_DURATION_DELTA_PERCENT` | `50` | Maximum allowed duration increase | +| `RULE_PERFORMANCE_DEFAULT_MAX_MEMORY_BYTES` | `268435456` | Absolute memory limit (256MB) | +| `RULE_PERFORMANCE_DEFAULT_MAX_MEMORY_DELTA_PERCENT` | `50` | Maximum allowed memory increase | +| `RULE_PERFORMANCE_DEFAULT_MAX_ERROR_RATE` | `0.05` | Maximum allowed failure ratio | +| `RULE_PERFORMANCE_WARNING_FACTOR` | `0.8` | Ratio of threshold triggering warning | +| `RULE_PERFORMANCE_MAX_SAMPLES_PER_RULE` | `1000` | Max samples stored per rule | +| `RULE_PERFORMANCE_BASELINE_RETENTION_MS` | `2592000000` | Retention window for baseline records (30 days) | + +--- + +## Failure Semantics & Resilience + +- **Kill switch**: Setting `RULE_PERFORMANCE_TRACKING_ENABLED=false` bypasses all regression checks gracefully, returning `status: "passed"` with zero overhead. +- **Partial rule failure**: If a specific rule throws or times out during worker execution, the worker isolates the failure, records status `"error"`, and continues executing the remainder of the rule-pack. +- **Store unavailability**: Storage read/write errors are classified as `STORE_UNAVAILABLE`, allowing queue retry policies to schedule backoff without corrupting scan outcomes. +- **Log sanitization**: All log events strip tokens, credentials, private keys, passwords, and file paths.