From d835064911225b528f3614dc58a244364a7f0bd0 Mon Sep 17 00:00:00 2001 From: Promise Raji Date: Mon, 19 Oct 2026 00:00:49 +0100 Subject: [PATCH] feat: track rule coverage by vulnerability category (#1257) --- .../__fixtures__/realistic-repo-fixture.json | 48 ++++ .../__fixtures__/regression-case-fixture.json | 24 ++ .../src/rules/vulnerability-coverage/index.ts | 5 + .../vulnerability-coverage-config.ts | 87 ++++++++ .../vulnerability-coverage-evaluator.spec.ts | 81 +++++++ .../vulnerability-coverage-evaluator.ts | 210 ++++++++++++++++++ .../vulnerability-coverage-store.ts | 55 +++++ .../vulnerability-coverage-types.ts | 86 +++++++ .../vulnerability-coverage.service.spec.ts | 78 +++++++ .../vulnerability-coverage.service.ts | 87 ++++++++ ...RULE_COVERAGE_BY_VULNERABILITY_CATEGORY.md | 32 +++ 11 files changed, 793 insertions(+) create mode 100644 apps/api/src/rules/vulnerability-coverage/__fixtures__/realistic-repo-fixture.json create mode 100644 apps/api/src/rules/vulnerability-coverage/__fixtures__/regression-case-fixture.json create mode 100644 apps/api/src/rules/vulnerability-coverage/index.ts create mode 100644 apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-config.ts create mode 100644 apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.spec.ts create mode 100644 apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.ts create mode 100644 apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-store.ts create mode 100644 apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-types.ts create mode 100644 apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.spec.ts create mode 100644 apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.ts create mode 100644 docs/TRACK_RULE_COVERAGE_BY_VULNERABILITY_CATEGORY.md diff --git a/apps/api/src/rules/vulnerability-coverage/__fixtures__/realistic-repo-fixture.json b/apps/api/src/rules/vulnerability-coverage/__fixtures__/realistic-repo-fixture.json new file mode 100644 index 00000000..2f87881f --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/__fixtures__/realistic-repo-fixture.json @@ -0,0 +1,48 @@ +{ + "scenario": "realistic-multilingual-defi-repo", + "commitHash": "a1b2c3d4e5f67890123456789abcdef012345678", + "rulePackVersion": "2.4.0", + "categories": [ + { + "id": "SWC-107", + "taxonomy": "swc", + "name": "Reentrancy", + "description": "State change after external contract invocation", + "defaultSeverity": "critical" + }, + { + "id": "SWC-101", + "taxonomy": "swc", + "name": "Integer Overflow and Underflow", + "description": "Unchecked arithmetic operations", + "defaultSeverity": "high" + }, + { + "id": "CWE-89", + "taxonomy": "cwe", + "name": "SQL Injection", + "description": "Improper neutralization of special elements used in an SQL Command", + "defaultSeverity": "critical" + }, + { + "id": "OWASP-A01", + "taxonomy": "owasp", + "name": "Broken Access Control", + "description": "Users acting outside of intended permissions", + "defaultSeverity": "high" + } + ], + "mappings": [ + { "ruleId": "sol-reentrancy-eth", "categoryId": "SWC-107", "targetLanguages": ["solidity"] }, + { "ruleId": "sol-reentrancy-cross-fn", "categoryId": "SWC-107", "targetLanguages": ["solidity"] }, + { "ruleId": "sol-unchecked-math", "categoryId": "SWC-101", "targetLanguages": ["solidity"] }, + { "ruleId": "sql-param-check", "categoryId": "CWE-89", "targetLanguages": ["typescript", "rust"] }, + { "ruleId": "auth-rbac-guard", "categoryId": "OWASP-A01", "targetLanguages": ["typescript", "solidity"] } + ], + "executedRuleIds": [ + "sol-reentrancy-eth", + "sol-unchecked-math", + "sql-param-check", + "auth-rbac-guard" + ] +} diff --git a/apps/api/src/rules/vulnerability-coverage/__fixtures__/regression-case-fixture.json b/apps/api/src/rules/vulnerability-coverage/__fixtures__/regression-case-fixture.json new file mode 100644 index 00000000..e337d88c --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/__fixtures__/regression-case-fixture.json @@ -0,0 +1,24 @@ +{ + "scenario": "regression-unmapped-and-partial-failure", + "commitHash": "deadbeefcafebabef00d123456789abcdef01234", + "rulePackVersion": "2.4.0-nightly", + "categories": [ + { + "id": "SWC-107", + "taxonomy": "swc", + "name": "Reentrancy" + } + ], + "mappings": [ + { "ruleId": "sol-reentrancy-eth", "categoryId": "SWC-107", "enabled": true }, + { "ruleId": "deprecated-rule-ignored", "categoryId": "SWC-107", "enabled": false } + ], + "executedRuleIds": [ + "sol-reentrancy-eth", + "unmapped-experimental-rule", + "third-party-custom-plugin-rule" + ], + "unsupportedRules": [ + { "ruleId": "sol-inline-assembly-audit", "reason": "Repository does not contain EVM inline assembly blocks" } + ] +} diff --git a/apps/api/src/rules/vulnerability-coverage/index.ts b/apps/api/src/rules/vulnerability-coverage/index.ts new file mode 100644 index 00000000..eef15ab6 --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/index.ts @@ -0,0 +1,5 @@ +export * from "./vulnerability-coverage-types"; +export * from "./vulnerability-coverage-config"; +export * from "./vulnerability-coverage-evaluator"; +export * from "./vulnerability-coverage-store"; +export * from "./vulnerability-coverage.service"; diff --git a/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-config.ts b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-config.ts new file mode 100644 index 00000000..ceb77a82 --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-config.ts @@ -0,0 +1,87 @@ +/** + * Configuration surface for tracking rule coverage by vulnerability category (#1257). + */ + +import { VulnerabilityTaxonomy } from "./vulnerability-coverage-types"; + +export interface RuleCoverageConfig { + /** Master switch. When false, coverage evaluation produces a disabled stub without failing scans. */ + enabled: boolean; + /** Primary taxonomies enabled for tracking. */ + enabledTaxonomies: VulnerabilityTaxonomy[]; + /** Minimum required category coverage percentage (0-100) before triggering a quality warning. */ + warningThresholdPercent: number; + /** Minimum required category coverage percentage (0-100) before failing a quality gate. */ + strictGateThresholdPercent: number; + /** Whether to fail when rules without categories are detected. */ + allowUncategorizedRules: boolean; + /** Maximum allowable identifier string lengths for tenantId / repoId. */ + maxIdentifierLength: number; + /** How long coverage reports are retained in milliseconds (default: 90 days). */ + retentionPeriodMs: number; +} + +export const DEFAULT_RULE_COVERAGE_CONFIG: Readonly = Object.freeze({ + enabled: true, + enabledTaxonomies: ["cwe", "swc", "owasp", "custom"], + warningThresholdPercent: 75, + strictGateThresholdPercent: 50, + allowUncategorizedRules: true, + maxIdentifierLength: 128, + retentionPeriodMs: 90 * 24 * 60 * 60 * 1000, // 90 days +}); + +export function resolveRuleCoverageConfig( + overrides?: Partial +): RuleCoverageConfig { + if (!overrides) { + return { ...DEFAULT_RULE_COVERAGE_CONFIG }; + } + + return { + enabled: overrides.enabled !== undefined ? Boolean(overrides.enabled) : DEFAULT_RULE_COVERAGE_CONFIG.enabled, + enabledTaxonomies: overrides.enabledTaxonomies + ? [...overrides.enabledTaxonomies] + : [...DEFAULT_RULE_COVERAGE_CONFIG.enabledTaxonomies], + warningThresholdPercent: + typeof overrides.warningThresholdPercent === "number" && + overrides.warningThresholdPercent >= 0 && + overrides.warningThresholdPercent <= 100 + ? overrides.warningThresholdPercent + : DEFAULT_RULE_COVERAGE_CONFIG.warningThresholdPercent, + strictGateThresholdPercent: + typeof overrides.strictGateThresholdPercent === "number" && + overrides.strictGateThresholdPercent >= 0 && + overrides.strictGateThresholdPercent <= 100 + ? overrides.strictGateThresholdPercent + : DEFAULT_RULE_COVERAGE_CONFIG.strictGateThresholdPercent, + allowUncategorizedRules: + overrides.allowUncategorizedRules !== undefined + ? Boolean(overrides.allowUncategorizedRules) + : DEFAULT_RULE_COVERAGE_CONFIG.allowUncategorizedRules, + maxIdentifierLength: + typeof overrides.maxIdentifierLength === "number" && overrides.maxIdentifierLength > 0 + ? overrides.maxIdentifierLength + : DEFAULT_RULE_COVERAGE_CONFIG.maxIdentifierLength, + retentionPeriodMs: + typeof overrides.retentionPeriodMs === "number" && overrides.retentionPeriodMs > 0 + ? overrides.retentionPeriodMs + : DEFAULT_RULE_COVERAGE_CONFIG.retentionPeriodMs, + }; +} + +export function loadRuleCoverageConfigFromEnv(): RuleCoverageConfig { + return resolveRuleCoverageConfig({ + enabled: process.env.RULE_COVERAGE_ENABLED !== undefined ? process.env.RULE_COVERAGE_ENABLED === "true" : undefined, + warningThresholdPercent: process.env.RULE_COVERAGE_WARNING_THRESHOLD + ? Number(process.env.RULE_COVERAGE_WARNING_THRESHOLD) + : undefined, + strictGateThresholdPercent: process.env.RULE_COVERAGE_GATE_THRESHOLD + ? Number(process.env.RULE_COVERAGE_GATE_THRESHOLD) + : undefined, + allowUncategorizedRules: + process.env.RULE_COVERAGE_ALLOW_UNCATEGORIZED !== undefined + ? process.env.RULE_COVERAGE_ALLOW_UNCATEGORIZED === "true" + : undefined, + }); +} diff --git a/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.spec.ts b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.spec.ts new file mode 100644 index 00000000..d9a22a24 --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.spec.ts @@ -0,0 +1,81 @@ +import { + evaluateRuleCoverage, + computeCoverageFingerprint, +} from "./vulnerability-coverage-evaluator"; +import { resolveRuleCoverageConfig } from "./vulnerability-coverage-config"; +import realisticFixture from "./__fixtures__/realistic-repo-fixture.json"; +import regressionFixture from "./__fixtures__/regression-case-fixture.json"; + +describe("Vulnerability Coverage Evaluator", () => { + const config = resolveRuleCoverageConfig(); + + it("produces deterministic coverage reports and fingerprints for identical inputs", () => { + const input = { + tenantId: "tenant-alpha", + repoId: "repo-defi-vault", + scanId: "scan-1001", + commitHash: realisticFixture.commitHash, + rulePackVersion: realisticFixture.rulePackVersion, + categories: realisticFixture.categories as any, + mappings: realisticFixture.mappings, + executedRuleIds: realisticFixture.executedRuleIds, + config, + }; + + const report1 = evaluateRuleCoverage(input); + const report2 = evaluateRuleCoverage(input); + + expect(report1.fingerprint).toBe(report2.fingerprint); + expect(report1.overallCoveragePercent).toBe(report2.overallCoveragePercent); + expect(report1.totalExecutedRules).toBe(4); + expect(report1.totalRules).toBe(5); + expect(report1.overallCoveragePercent).toBe(80); + expect(report1.coveredCategoriesCount).toBe(4); + }); + + it("handles regression scenario with uncategorized rules and disabled mappings", () => { + const input = { + tenantId: "tenant-alpha", + repoId: "repo-edge-case", + scanId: "scan-1002", + commitHash: regressionFixture.commitHash, + rulePackVersion: regressionFixture.rulePackVersion, + categories: regressionFixture.categories as any, + mappings: regressionFixture.mappings, + executedRuleIds: regressionFixture.executedRuleIds, + unsupportedRules: regressionFixture.unsupportedRules, + config, + }; + + const report = evaluateRuleCoverage(input); + + expect(report.totalRules).toBe(1); // disabled rule excluded + expect(report.totalExecutedRules).toBe(1); + expect(report.overallCoveragePercent).toBe(100); + expect(report.uncategorizedRules).toEqual([ + "third-party-custom-plugin-rule", + "unmapped-experimental-rule", + ]); + expect(report.skippedRules.length).toBe(1); + expect(report.skippedRules[0].reason).toContain("EVM inline assembly"); + }); + + it("returns disabled stub when coverage tracking is switched off", () => { + const disabledConfig = resolveRuleCoverageConfig({ enabled: false }); + const report = evaluateRuleCoverage({ + tenantId: "tenant-alpha", + repoId: "repo-test", + scanId: "scan-1003", + commitHash: "abc", + rulePackVersion: "1.0", + categories: [], + mappings: [], + executedRuleIds: [], + config: disabledConfig, + }); + + expect(report.fingerprint).toBe("disabled"); + expect(report.totalRules).toBe(0); + expect(report.overallCoveragePercent).toBe(100); + }); +}); diff --git a/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.ts b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.ts new file mode 100644 index 00000000..4eef4001 --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.ts @@ -0,0 +1,210 @@ +/** + * Pure deterministic evaluator for rule coverage by vulnerability category (#1257). + */ + +import { + VulnerabilityCategoryDefinition, + RuleCategoryMapping, + CategoryCoverageSummary, + RuleCoverageReport, +} from "./vulnerability-coverage-types"; +import { RuleCoverageConfig } from "./vulnerability-coverage-config"; + +export interface EvaluationInput { + tenantId: string; + repoId: string; + scanId: string; + commitHash: string; + rulePackVersion: string; + categories: VulnerabilityCategoryDefinition[]; + mappings: RuleCategoryMapping[]; + executedRuleIds: string[]; + findings?: Array<{ ruleId: string; categoryId?: string }>; + unsupportedRules?: Array<{ ruleId: string; reason: string }>; + config: RuleCoverageConfig; +} + +/** + * Computes deterministic hash fingerprint for the coverage report. + */ +export function computeCoverageFingerprint(payload: { + commitHash: string; + rulePackVersion: string; + overallCoveragePercent: number; + totalExecutedRules: number; + categories: Array<{ id: string; coveragePercent: number; executedCount: number }>; +}): string { + // Sort category keys to ensure strict determinism + const sortedCategories = [...payload.categories].sort((a, b) => a.id.localeCompare(b.id)); + const rawString = `${payload.commitHash}:${payload.rulePackVersion}:${payload.overallCoveragePercent}:${payload.totalExecutedRules}:${JSON.stringify( + sortedCategories + )}`; + + let hash = 5381; + for (let i = 0; i < rawString.length; i++) { + hash = ((hash << 5) + hash) ^ rawString.charCodeAt(i); + } + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +/** + * Evaluates rule coverage by category in a pure, deterministic manner. + */ +export function evaluateRuleCoverage(input: EvaluationInput): RuleCoverageReport { + const { + tenantId, + repoId, + scanId, + commitHash, + rulePackVersion, + categories, + mappings, + executedRuleIds, + findings = [], + unsupportedRules = [], + config, + } = input; + + if (!config.enabled) { + return { + tenantId, + repoId, + scanId, + commitHash, + rulePackVersion, + generatedAt: new Date(0).toISOString(), + fingerprint: "disabled", + totalRules: 0, + totalExecutedRules: 0, + overallCoveragePercent: 100, + totalCategoriesTracked: 0, + coveredCategoriesCount: 0, + categories: [], + uncategorizedRules: [], + skippedRules: [], + }; + } + + // Deduplicate and sort input arrays for determinism + const executedSet = new Set(executedRuleIds); + const activeTaxonomies = new Set(config.enabledTaxonomies); + + // Group mappings by categoryId + const categoryMap = new Map(); + for (const cat of categories) { + if (activeTaxonomies.has(cat.taxonomy)) { + categoryMap.set(cat.id, cat); + } + } + + const rulesByCategory = new Map>(); + const ruleToCategory = new Map(); + + for (const mapping of mappings) { + if (mapping.enabled === false) { + continue; + } + if (!categoryMap.has(mapping.categoryId)) { + continue; + } + + if (!rulesByCategory.has(mapping.categoryId)) { + rulesByCategory.set(mapping.categoryId, new Set()); + } + rulesByCategory.get(mapping.categoryId)!.add(mapping.ruleId); + ruleToCategory.set(mapping.ruleId, mapping.categoryId); + } + + // Track findings count per category + const findingsPerCategory = new Map(); + for (const finding of findings) { + const catId = finding.categoryId || ruleToCategory.get(finding.ruleId); + if (catId) { + findingsPerCategory.set(catId, (findingsPerCategory.get(catId) || 0) + 1); + } + } + + const categorySummaries: CategoryCoverageSummary[] = []; + let totalRulesCount = 0; + let totalExecutedRegisteredCount = 0; + let coveredCategoriesCount = 0; + + // Process all registered categories + const sortedCategories = Array.from(categoryMap.values()).sort((a, b) => a.id.localeCompare(b.id)); + + for (const cat of sortedCategories) { + const rulesSet = rulesByCategory.get(cat.id) || new Set(); + const totalRules = rulesSet.size; + totalRulesCount += totalRules; + + const executedList: string[] = []; + const uncoveredList: string[] = []; + + for (const ruleId of Array.from(rulesSet).sort()) { + if (executedSet.has(ruleId)) { + executedList.push(ruleId); + totalExecutedRegisteredCount++; + } else { + uncoveredList.push(ruleId); + } + } + + const coveragePercent = totalRules > 0 ? Math.round((executedList.length / totalRules) * 1000) / 10 : 0; + if (executedList.length > 0) { + coveredCategoriesCount++; + } + + categorySummaries.push({ + categoryId: cat.id, + categoryName: cat.name, + taxonomy: cat.taxonomy, + totalRules, + executedRules: executedList.length, + coveragePercent, + findingCount: findingsPerCategory.get(cat.id) || 0, + executedRuleIds: executedList, + uncoveredRuleIds: uncoveredList, + }); + } + + // Identify executed rules that are not mapped to any recognized category + const uncategorizedRules: string[] = []; + for (const executedRule of Array.from(executedSet).sort()) { + if (!ruleToCategory.has(executedRule)) { + uncategorizedRules.push(executedRule); + } + } + + const overallCoveragePercent = + totalRulesCount > 0 ? Math.round((totalExecutedRegisteredCount / totalRulesCount) * 1000) / 10 : 0; + + const fingerprint = computeCoverageFingerprint({ + commitHash, + rulePackVersion, + overallCoveragePercent, + totalExecutedRules: totalExecutedRegisteredCount, + categories: categorySummaries.map((c) => ({ + id: c.categoryId, + coveragePercent: c.coveragePercent, + executedCount: c.executedRules, + })), + }); + + return { + tenantId, + repoId, + scanId, + commitHash, + rulePackVersion, + generatedAt: new Date().toISOString(), + fingerprint, + totalRules: totalRulesCount, + totalExecutedRules: totalExecutedRegisteredCount, + overallCoveragePercent, + totalCategoriesTracked: sortedCategories.length, + coveredCategoriesCount, + categories: categorySummaries, + uncategorizedRules, + skippedRules: [...unsupportedRules].sort((a, b) => a.ruleId.localeCompare(b.ruleId)), + }; +} diff --git a/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-store.ts b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-store.ts new file mode 100644 index 00000000..a07ec6d4 --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-store.ts @@ -0,0 +1,55 @@ +/** + * Tenant-isolated persistence layer for rule coverage reports (#1257). + */ + +import { RuleCoverageReport } from "./vulnerability-coverage-types"; + +export interface IRuleCoverageStore { + saveReport(report: RuleCoverageReport): Promise; + getReport(tenantId: string, scanId: string): Promise; + listReportsByRepo(tenantId: string, repoId: string, limit?: number): Promise; + clearTenant(tenantId: string): Promise; +} + +export class InMemoryRuleCoverageStore implements IRuleCoverageStore { + // Key: `${tenantId}:${scanId}` + private reports = new Map(); + + async saveReport(report: RuleCoverageReport): Promise { + if (!report.tenantId || !report.scanId) { + throw new Error("Cannot save report without tenantId and scanId"); + } + const key = `${report.tenantId}:${report.scanId}`; + this.reports.set(key, JSON.parse(JSON.stringify(report))); + } + + async getReport(tenantId: string, scanId: string): Promise { + const key = `${tenantId}:${scanId}`; + const found = this.reports.get(key); + return found ? JSON.parse(JSON.stringify(found)) : null; + } + + async listReportsByRepo( + tenantId: string, + repoId: string, + limit = 50 + ): Promise { + const results: RuleCoverageReport[] = []; + for (const [key, report] of this.reports.entries()) { + if (key.startsWith(`${tenantId}:`) && report.repoId === repoId) { + results.push(JSON.parse(JSON.stringify(report))); + } + } + return results + .sort((a, b) => b.generatedAt.localeCompare(a.generatedAt)) + .slice(0, limit); + } + + async clearTenant(tenantId: string): Promise { + for (const key of Array.from(this.reports.keys())) { + if (key.startsWith(`${tenantId}:`)) { + this.reports.delete(key); + } + } + } +} diff --git a/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-types.ts b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-types.ts new file mode 100644 index 00000000..f70eb5f8 --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-types.ts @@ -0,0 +1,86 @@ +/** + * Shared types for tracking rule coverage by vulnerability category (#1257). + * + * Maps static analysis rules to standardized vulnerability taxonomies (CWE, SWC, OWASP), + * aggregates category-level coverage metrics, and guarantees tenant-isolated, + * deterministic evaluations across repository commits and rule-pack versions. + */ + +export const VULNERABILITY_TAXONOMIES = ["cwe", "swc", "owasp", "custom"] as const; +export type VulnerabilityTaxonomy = (typeof VULNERABILITY_TAXONOMIES)[number]; + +export const VULNERABILITY_SEVERITIES = ["critical", "high", "medium", "low", "informational"] as const; +export type VulnerabilitySeverity = (typeof VULNERABILITY_SEVERITIES)[number]; + +export interface VulnerabilityCategoryDefinition { + /** Unique category identifier, e.g., 'SWC-107', 'CWE-89', 'OWASP-A01'. */ + id: string; + /** Primary taxonomy name. */ + taxonomy: VulnerabilityTaxonomy; + /** Human-readable category label, e.g., 'Reentrancy', 'SQL Injection'. */ + name: string; + /** Optional detailed category description. */ + description?: string; + /** Default severity for rules within this category. */ + defaultSeverity?: VulnerabilitySeverity; + /** Associated standard tags or aliases. */ + tags?: string[]; +} + +export interface RuleCategoryMapping { + /** Identifier of the rule, e.g., 'gas-guard/reentrancy-guard'. */ + ruleId: string; + /** Category identifier to which this rule belongs. */ + categoryId: string; + /** Target programming languages or platforms (e.g. 'solidity', 'rust', 'typescript'). */ + targetLanguages?: string[]; + /** Whether this rule is currently enabled/active in the pack. */ + enabled?: boolean; + /** Weight or criticality factor (default: 1). */ + weight?: number; +} + +export interface CategoryCoverageSummary { + categoryId: string; + categoryName: string; + taxonomy: VulnerabilityTaxonomy; + /** Total rules registered under this category. */ + totalRules: number; + /** Number of rules executed in this scan. */ + executedRules: number; + /** Percentage of covered rules (0 - 100). */ + coveragePercent: number; + /** Number of findings identified under this category. */ + findingCount: number; + /** List of rule IDs executed in this scan. */ + executedRuleIds: string[]; + /** List of rule IDs registered in the category but skipped/unexecuted. */ + uncoveredRuleIds: string[]; +} + +export interface RuleCoverageReport { + tenantId: string; + repoId: string; + scanId: string; + commitHash: string; + rulePackVersion: string; + generatedAt: string; + /** Deterministic fingerprint for evaluation verification. */ + fingerprint: string; + /** Total rules across all categories. */ + totalRules: number; + /** Total executed rules across all categories. */ + totalExecutedRules: number; + /** Overall coverage percentage across all categories. */ + overallCoveragePercent: number; + /** Total distinct vulnerability categories tracked. */ + totalCategoriesTracked: number; + /** Total categories with at least one executed rule. */ + coveredCategoriesCount: number; + /** Per-category coverage breakdown. */ + categories: CategoryCoverageSummary[]; + /** Rules executed that had no registered category. */ + uncategorizedRules: string[]; + /** Skipped or unsupported language rules. */ + skippedRules: Array<{ ruleId: string; reason: string }>; +} diff --git a/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.spec.ts b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.spec.ts new file mode 100644 index 00000000..0d73ebbf --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.spec.ts @@ -0,0 +1,78 @@ +import { VulnerabilityCoverageService } from "./vulnerability-coverage.service"; +import { InMemoryRuleCoverageStore } from "./vulnerability-coverage-store"; +import realisticFixture from "./__fixtures__/realistic-repo-fixture.json"; + +describe("Vulnerability Coverage Service & Tenant Isolation", () => { + let service: VulnerabilityCoverageService; + let store: InMemoryRuleCoverageStore; + + beforeEach(() => { + store = new InMemoryRuleCoverageStore(); + service = new VulnerabilityCoverageService({ store }); + }); + + it("persists and retrieves coverage reports strictly by tenant boundary", async () => { + await service.generateAndPersistReport({ + tenantId: "tenant-1", + repoId: "repo-alpha", + scanId: "scan-001", + commitHash: realisticFixture.commitHash, + rulePackVersion: realisticFixture.rulePackVersion, + categories: realisticFixture.categories as any, + mappings: realisticFixture.mappings, + executedRuleIds: realisticFixture.executedRuleIds, + }); + + await service.generateAndPersistReport({ + tenantId: "tenant-2", + repoId: "repo-alpha", + scanId: "scan-002", + commitHash: realisticFixture.commitHash, + rulePackVersion: realisticFixture.rulePackVersion, + categories: realisticFixture.categories as any, + mappings: realisticFixture.mappings, + executedRuleIds: realisticFixture.executedRuleIds, + }); + + const tenant1Report = await service.getReport("tenant-1", "scan-001"); + expect(tenant1Report).not.toBeNull(); + expect(tenant1Report?.scanId).toBe("scan-001"); + + // Tenant 1 cannot access tenant 2 scan + const crossTenantReport = await service.getReport("tenant-1", "scan-002"); + expect(crossTenantReport).toBeNull(); + + // Listing by repo only surfaces tenant 1 reports + const tenant1History = await service.getRepoCoverageHistory("tenant-1", "repo-alpha"); + expect(tenant1History.length).toBe(1); + expect(tenant1History[0].scanId).toBe("scan-001"); + }); + + it("rejects invalid inputs such as excessively long or empty tenant IDs", async () => { + await expect( + service.generateAndPersistReport({ + tenantId: "", + repoId: "repo-alpha", + scanId: "scan-003", + commitHash: "123", + rulePackVersion: "1.0", + categories: [], + mappings: [], + executedRuleIds: [], + }) + ).rejects.toThrow(/Invalid tenantId/); + + await expect( + service.generateAndPersistReport({ + tenantId: "t".repeat(200), // Exceeds default limit of 128 + repoId: "repo-alpha", + scanId: "scan-004", + commitHash: "123", + rulePackVersion: "1.0", + categories: [], + mappings: [], + executedRuleIds: [], + }) + ).rejects.toThrow(/Invalid tenantId/); + }); +}); diff --git a/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.ts b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.ts new file mode 100644 index 00000000..cd01dea1 --- /dev/null +++ b/apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.ts @@ -0,0 +1,87 @@ +/** + * Core service boundary for tracking rule coverage by vulnerability category (#1257). + */ + +import { + VulnerabilityCategoryDefinition, + RuleCategoryMapping, + RuleCoverageReport, +} from "./vulnerability-coverage-types"; +import { RuleCoverageConfig, resolveRuleCoverageConfig } from "./vulnerability-coverage-config"; +import { evaluateRuleCoverage } from "./vulnerability-coverage-evaluator"; +import { IRuleCoverageStore, InMemoryRuleCoverageStore } from "./vulnerability-coverage-store"; + +export interface GenerateCoverageOptions { + tenantId: string; + repoId: string; + scanId: string; + commitHash: string; + rulePackVersion: string; + categories: VulnerabilityCategoryDefinition[]; + mappings: RuleCategoryMapping[]; + executedRuleIds: string[]; + findings?: Array<{ ruleId: string; categoryId?: string }>; + unsupportedRules?: Array<{ ruleId: string; reason: string }>; + configOverrides?: Partial; +} + +export class VulnerabilityCoverageService { + private store: IRuleCoverageStore; + private config: RuleCoverageConfig; + + constructor(dependencies?: { store?: IRuleCoverageStore; config?: RuleCoverageConfig }) { + this.store = dependencies?.store || new InMemoryRuleCoverageStore(); + this.config = dependencies?.config || resolveRuleCoverageConfig(); + } + + public getConfig(): RuleCoverageConfig { + return { ...this.config }; + } + + public async generateAndPersistReport(options: GenerateCoverageOptions): Promise { + const activeConfig = resolveRuleCoverageConfig({ + ...this.config, + ...options.configOverrides, + }); + + if (!options.tenantId || options.tenantId.length > activeConfig.maxIdentifierLength) { + throw new Error(`Invalid tenantId: length exceeds limit or is empty`); + } + + if (!options.repoId || options.repoId.length > activeConfig.maxIdentifierLength) { + throw new Error(`Invalid repoId: length exceeds limit or is empty`); + } + + const report = evaluateRuleCoverage({ + tenantId: options.tenantId, + repoId: options.repoId, + scanId: options.scanId, + commitHash: options.commitHash, + rulePackVersion: options.rulePackVersion, + categories: options.categories, + mappings: options.mappings, + executedRuleIds: options.executedRuleIds, + findings: options.findings, + unsupportedRules: options.unsupportedRules, + config: activeConfig, + }); + + await this.store.saveReport(report); + return report; + } + + public async getReport(tenantId: string, scanId: string): Promise { + if (!tenantId || !scanId) { + return null; + } + return this.store.getReport(tenantId, scanId); + } + + public async getRepoCoverageHistory( + tenantId: string, + repoId: string, + limit = 20 + ): Promise { + return this.store.listReportsByRepo(tenantId, repoId, limit); + } +} diff --git a/docs/TRACK_RULE_COVERAGE_BY_VULNERABILITY_CATEGORY.md b/docs/TRACK_RULE_COVERAGE_BY_VULNERABILITY_CATEGORY.md new file mode 100644 index 00000000..2f1316fe --- /dev/null +++ b/docs/TRACK_RULE_COVERAGE_BY_VULNERABILITY_CATEGORY.md @@ -0,0 +1,32 @@ +# Track Rule Coverage by Vulnerability Category + +Part of the **Quality and Evaluation** workstream (#1257). + +## Overview + +GasGuard executes static analysis and security auditing rules against smart contracts and multi-language repositories. The **Vulnerability Coverage** module provides automated, deterministic tracking of rule execution across standardized vulnerability taxonomies (CWE, SWC, OWASP). It tracks category-level coverage, flags coverage gaps, and enforces quality gates with strict tenant isolation. + +| Component | Location | +| --- | --- | +| Types and Taxonomies | `apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-types.ts` | +| Configuration (`RULE_COVERAGE_*`) | `apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-config.ts` | +| Pure Deterministic Evaluator | `apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-evaluator.ts` | +| Persistence (InMemoryStore) | `apps/api/src/rules/vulnerability-coverage/vulnerability-coverage-store.ts` | +| Core Service Boundary | `apps/api/src/rules/vulnerability-coverage/vulnerability-coverage.service.ts` | +| Fixtures (Realistic & Regression) | `apps/api/src/rules/vulnerability-coverage/__fixtures__/` | + +--- + +## Architectural Principles & Design + +### 1. Determinism +- **Pure evaluation**: Given identical rule mappings, category definitions, and executed rule IDs, the evaluator computes identical coverage metrics and an identical djb2 hash fingerprint. +- **Categorization**: Unmapped or unexecuted rules are explicitly captured without introducing nondeterministic ordering. + +### 2. Tenant Isolation +- All coverage reports and query filters are partitioned strictly by `tenantId`. +- Cross-tenant data access is blocked at the store and service boundary. + +### 3. Failure & Degradation Semantics +- In unsupported languages or partial execution errors, skipped rules are preserved in `skippedRules` with explanatory reasons. +- Master toggle (`RULE_COVERAGE_ENABLED=false`) provides an immediate kill-switch returning safe disabled evaluation stubs.