From dfd713b1ef5ed4162694cbf37c59ae7a699f0a4f Mon Sep 17 00:00:00 2001 From: software321dev Date: Sun, 18 Oct 2026 18:17:56 +0100 Subject: [PATCH] feat(observability): scan stage histograms, analyzer saturation, failure taxonomy, and throughput dashboard - Issue #1259: Add scan stage duration histograms with configurable buckets - Issue #1260: Expose analyzer saturation and queue pressure metrics - Issue #1261: Create structured scan failure taxonomy with retryable classification - Issue #1264: Add operational dashboard for scan throughput and files-per-second metrics Closes #1259, Closes #1260, Closes #1261, Closes #1264 --- .../scan-throughput-dashboard.service.ts | 55 +++++++ .../analyzer-saturation-metrics.service.ts | 58 +++++++ .../metrics/scan-stage-histogram.service.ts | 66 ++++++++ src/observability/scan-observability.spec.ts | 143 ++++++++++++++++++ .../taxonomy/scan-failure-taxonomy.service.ts | 78 ++++++++++ .../types/scan-observability.types.ts | 78 ++++++++++ 6 files changed, 478 insertions(+) create mode 100644 src/observability/dashboard/scan-throughput-dashboard.service.ts create mode 100644 src/observability/metrics/analyzer-saturation-metrics.service.ts create mode 100644 src/observability/metrics/scan-stage-histogram.service.ts create mode 100644 src/observability/scan-observability.spec.ts create mode 100644 src/observability/taxonomy/scan-failure-taxonomy.service.ts create mode 100644 src/observability/types/scan-observability.types.ts diff --git a/src/observability/dashboard/scan-throughput-dashboard.service.ts b/src/observability/dashboard/scan-throughput-dashboard.service.ts new file mode 100644 index 00000000..9843da8a --- /dev/null +++ b/src/observability/dashboard/scan-throughput-dashboard.service.ts @@ -0,0 +1,55 @@ +import { + ScanThroughputRecord, + ThroughputDashboardSummary, +} from '../types/scan-observability.types'; + +export class ScanThroughputDashboardService { + private records: ScanThroughputRecord[] = []; + + public recordScan(record: ScanThroughputRecord): void { + this.records.push(record); + } + + public getSummary( + windowMs = 3600000, + tenantId?: string, + ): ThroughputDashboardSummary { + const now = Date.now(); + const startTime = now - windowMs; + + let relevant = this.records.filter((r) => r.timestamp >= startTime); + if (tenantId) { + relevant = relevant.filter((r) => r.tenantId === tenantId); + } + + const totalScans = relevant.length; + const successfulScans = relevant.filter((r) => r.success).length; + const failedScans = totalScans - successfulScans; + const successRate = totalScans > 0 ? (successfulScans / totalScans) * 100 : 0; + const totalFilesScanned = relevant.reduce((sum, r) => sum + r.filesScanned, 0); + const totalDurationMs = relevant.reduce((sum, r) => sum + r.durationMs, 0); + + const windowMinutes = Math.max(1, windowMs / 60000); + const windowSeconds = Math.max(1, windowMs / 1000); + + const scansPerMinute = Number((totalScans / windowMinutes).toFixed(2)); + const filesPerSecond = Number((totalFilesScanned / windowSeconds).toFixed(2)); + const avgDurationMs = totalScans > 0 ? Math.round(totalDurationMs / totalScans) : 0; + + return { + windowMs, + totalScans, + successfulScans, + failedScans, + successRate: Number(successRate.toFixed(2)), + totalFilesScanned, + scansPerMinute, + filesPerSecond, + avgDurationMs, + }; + } + + public clear(): void { + this.records = []; + } +} diff --git a/src/observability/metrics/analyzer-saturation-metrics.service.ts b/src/observability/metrics/analyzer-saturation-metrics.service.ts new file mode 100644 index 00000000..74aea413 --- /dev/null +++ b/src/observability/metrics/analyzer-saturation-metrics.service.ts @@ -0,0 +1,58 @@ +import { QueuePressureMetrics } from '../types/scan-observability.types'; + +export class AnalyzerSaturationMetricsService { + private activeWorkers = 0; + private queuedJobs: { jobId: string; queuedAt: number }[] = []; + private maxWorkers: number; + + constructor(maxWorkers = 10) { + this.maxWorkers = maxWorkers; + } + + public jobEnqueued(jobId: string, timestamp = Date.now()): void { + this.queuedJobs.push({ jobId, queuedAt: timestamp }); + } + + public jobStarted(jobId: string): number { + const index = this.queuedJobs.findIndex((j) => j.jobId === jobId); + let waitTimeMs = 0; + if (index !== -1) { + waitTimeMs = Date.now() - this.queuedJobs[index].queuedAt; + this.queuedJobs.splice(index, 1); + } + this.activeWorkers += 1; + return waitTimeMs; + } + + public jobFinished(): void { + if (this.activeWorkers > 0) { + this.activeWorkers -= 1; + } + } + + public getMetrics(): QueuePressureMetrics { + const now = Date.now(); + const totalWaitMs = this.queuedJobs.reduce( + (sum, j) => sum + (now - j.queuedAt), + 0, + ); + const averageWaitTimeMs = + this.queuedJobs.length > 0 ? totalWaitMs / this.queuedJobs.length : 0; + const saturationRatio = this.activeWorkers / Math.max(1, this.maxWorkers); + + return { + activeWorkers: this.activeWorkers, + maxWorkers: this.maxWorkers, + queuedJobs: this.queuedJobs.length, + saturationRatio: Number(saturationRatio.toFixed(3)), + isSaturated: saturationRatio >= 0.9, + averageWaitTimeMs: Math.round(averageWaitTimeMs), + timestamp: now, + }; + } + + public clear(): void { + this.activeWorkers = 0; + this.queuedJobs = []; + } +} diff --git a/src/observability/metrics/scan-stage-histogram.service.ts b/src/observability/metrics/scan-stage-histogram.service.ts new file mode 100644 index 00000000..b2e0553c --- /dev/null +++ b/src/observability/metrics/scan-stage-histogram.service.ts @@ -0,0 +1,66 @@ +import { + ScanStage, + StageDurationHistogram, +} from '../types/scan-observability.types'; + +export const DEFAULT_DURATION_BUCKETS = [ + 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, +]; + +export class ScanStageHistogramService { + private histograms = new Map< + ScanStage, + { count: number; sumMs: number; buckets: Map } + >(); + private bucketBoundaries: number[]; + + constructor(bucketBoundaries: number[] = DEFAULT_DURATION_BUCKETS) { + this.bucketBoundaries = [...bucketBoundaries].sort((a, b) => a - b); + } + + public recordStageDuration(stage: ScanStage, durationMs: number): void { + let stageData = this.histograms.get(stage); + if (!stageData) { + stageData = { + count: 0, + sumMs: 0, + buckets: new Map(), + }; + for (const le of this.bucketBoundaries) { + stageData.buckets.set(le, 0); + } + this.histograms.set(stage, stageData); + } + + stageData.count += 1; + stageData.sumMs += durationMs; + + for (const le of this.bucketBoundaries) { + if (durationMs <= le) { + const current = stageData.buckets.get(le) ?? 0; + stageData.buckets.set(le, current + 1); + } + } + } + + public getHistogram(stage: ScanStage): StageDurationHistogram | null { + const stageData = this.histograms.get(stage); + if (!stageData) return null; + + const buckets = this.bucketBoundaries.map((le) => ({ + le, + count: stageData.buckets.get(le) ?? 0, + })); + + return { + stage, + count: stageData.count, + sumMs: stageData.sumMs, + buckets, + }; + } + + public clear(): void { + this.histograms.clear(); + } +} diff --git a/src/observability/scan-observability.spec.ts b/src/observability/scan-observability.spec.ts new file mode 100644 index 00000000..068af3e8 --- /dev/null +++ b/src/observability/scan-observability.spec.ts @@ -0,0 +1,143 @@ +import { ScanStageHistogramService } from './metrics/scan-stage-histogram.service'; +import { AnalyzerSaturationMetricsService } from './metrics/analyzer-saturation-metrics.service'; +import { ScanFailureTaxonomyService } from './taxonomy/scan-failure-taxonomy.service'; +import { ScanThroughputDashboardService } from './dashboard/scan-throughput-dashboard.service'; + +describe('Scan Observability Services (#1259, #1260, #1261, #1264)', () => { + describe('ScanStageHistogramService (#1259)', () => { + let service: ScanStageHistogramService; + + beforeEach(() => { + service = new ScanStageHistogramService([100, 500, 1000]); + }); + + it('should record durations and increment correct histogram buckets', () => { + service.recordStageDuration('ast_generation', 50); + service.recordStageDuration('ast_generation', 300); + service.recordStageDuration('ast_generation', 800); + + const hist = service.getHistogram('ast_generation'); + expect(hist).not.toBeNull(); + expect(hist?.count).toBe(3); + expect(hist?.sumMs).toBe(1150); + + // Bucket <= 100 should have 1 item (50) + const b100 = hist?.buckets.find((b) => b.le === 100); + expect(b100?.count).toBe(1); + + // Bucket <= 500 should have 2 items (50, 300) + const b500 = hist?.buckets.find((b) => b.le === 500); + expect(b500?.count).toBe(2); + + // Bucket <= 1000 should have 3 items (50, 300, 800) + const b1000 = hist?.buckets.find((b) => b.le === 1000); + expect(b1000?.count).toBe(3); + }); + }); + + describe('AnalyzerSaturationMetricsService (#1260)', () => { + let service: AnalyzerSaturationMetricsService; + + beforeEach(() => { + service = new AnalyzerSaturationMetricsService(5); + }); + + it('should track active workers, queue wait time and saturation ratio', () => { + service.jobEnqueued('job-1', Date.now() - 2000); + service.jobEnqueued('job-2', Date.now() - 1000); + + const wait = service.jobStarted('job-1'); + expect(wait).toBeGreaterThanOrEqual(1900); + + const metrics = service.getMetrics(); + expect(metrics.activeWorkers).toBe(1); + expect(metrics.maxWorkers).toBe(5); + expect(metrics.queuedJobs).toBe(1); + expect(metrics.saturationRatio).toBe(0.2); + expect(metrics.isSaturated).toBe(false); + + service.jobFinished(); + const updated = service.getMetrics(); + expect(updated.activeWorkers).toBe(0); + }); + }); + + describe('ScanFailureTaxonomyService (#1261)', () => { + let service: ScanFailureTaxonomyService; + + beforeEach(() => { + service = new ScanFailureTaxonomyService(); + }); + + it('should classify failures into structured taxonomy categories', () => { + const f1 = service.classifyFailure( + 'scan-1', + 'tenant-a', + new Error('Unexpected token in line 5: syntax error'), + 'ast_generation', + ); + expect(f1.category).toBe('SYNTAX_ERROR'); + expect(f1.retryable).toBe(false); + + const f2 = service.classifyFailure( + 'scan-2', + 'tenant-a', + new Error('Operation timeout: deadline exceeded'), + 'cloning', + ); + expect(f2.category).toBe('TIMEOUT'); + expect(f2.retryable).toBe(true); + + const f3 = service.classifyFailure( + 'scan-3', + 'tenant-b', + new Error('Heap allocation failed: JavaScript out of memory'), + 'rule_evaluation', + ); + expect(f3.category).toBe('RESOURCE_EXHAUSTION'); + + const tenantFailures = service.getFailuresByTenant('tenant-a'); + expect(tenantFailures.length).toBe(2); + }); + }); + + describe('ScanThroughputDashboardService (#1264)', () => { + let service: ScanThroughputDashboardService; + + beforeEach(() => { + service = new ScanThroughputDashboardService(); + }); + + it('should calculate scan throughput, files per second and success rate', () => { + const now = Date.now(); + service.recordScan({ + scanId: 's-1', + tenantId: 'tenant-1', + repository: 'repo-a', + filesScanned: 100, + durationMs: 2000, + timestamp: now, + success: true, + }); + + service.recordScan({ + scanId: 's-2', + tenantId: 'tenant-1', + repository: 'repo-b', + filesScanned: 50, + durationMs: 1000, + timestamp: now, + success: false, + }); + + const summary = service.getSummary(60000, 'tenant-1'); + expect(summary.totalScans).toBe(2); + expect(summary.successfulScans).toBe(1); + expect(summary.failedScans).toBe(1); + expect(summary.successRate).toBe(50); + expect(summary.totalFilesScanned).toBe(150); + expect(summary.scansPerMinute).toBe(2); + expect(summary.avgDurationMs).toBe(1500); + }); + }); +}); diff --git a/src/observability/taxonomy/scan-failure-taxonomy.service.ts b/src/observability/taxonomy/scan-failure-taxonomy.service.ts new file mode 100644 index 00000000..9c6dc1fe --- /dev/null +++ b/src/observability/taxonomy/scan-failure-taxonomy.service.ts @@ -0,0 +1,78 @@ +import { + StructuredScanFailure, + ScanFailureCategory, + ScanStage, +} from '../types/scan-observability.types'; + +export class ScanFailureTaxonomyService { + private failures: StructuredScanFailure[] = []; + + public classifyFailure( + scanId: string, + tenantId: string, + error: Error | unknown, + stage: ScanStage, + context?: Record, + ): StructuredScanFailure { + const failureId = `fail-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`; + const message = error instanceof Error ? error.message : String(error); + const { category, errorCode, retryable } = this.resolveCategory(error, message, stage); + + const failure: StructuredScanFailure = { + failureId, + scanId, + tenantId, + category, + errorCode, + message, + stage, + retryable, + timestamp: Date.now(), + context, + }; + + this.failures.push(failure); + return failure; + } + + private resolveCategory( + error: unknown, + message: string, + stage: ScanStage, + ): { category: ScanFailureCategory; errorCode: string; retryable: boolean } { + const msg = message.toLowerCase(); + + if (msg.includes('syntax') || msg.includes('parse error')) { + return { category: 'SYNTAX_ERROR', errorCode: 'SCAN_SYNTAX_ERR', retryable: false }; + } + if (msg.includes('timeout') || msg.includes('deadline exceeded')) { + return { category: 'TIMEOUT', errorCode: 'SCAN_TIMEOUT', retryable: true }; + } + if (msg.includes('out of memory') || msg.includes('heap') || msg.includes('resource')) { + return { category: 'RESOURCE_EXHAUSTION', errorCode: 'SCAN_OOM', retryable: false }; + } + if (msg.includes('dependency') || msg.includes('cannot find package')) { + return { category: 'DEPENDENCY_RESOLVE_ERROR', errorCode: 'SCAN_DEP_MISSING', retryable: false }; + } + if (msg.includes('network') || msg.includes('econnreset') || msg.includes('etimedout')) { + return { category: 'NETWORK_IO_ERROR', errorCode: 'SCAN_NET_IO', retryable: true }; + } + if (stage === 'rule_evaluation') { + return { category: 'RULE_EXECUTION_FAILURE', errorCode: 'SCAN_RULE_ERR', retryable: false }; + } + + return { category: 'INTERNAL_ERROR', errorCode: 'SCAN_INTERNAL_ERR', retryable: true }; + } + + public getFailuresByTenant(tenantId: string): StructuredScanFailure[] { + return this.failures.filter((f) => f.tenantId === tenantId); + } + + public getFailuresByCategory(category: ScanFailureCategory): StructuredScanFailure[] { + return this.failures.filter((f) => f.category === category); + } + + public clear(): void { + this.failures = []; + } +} diff --git a/src/observability/types/scan-observability.types.ts b/src/observability/types/scan-observability.types.ts new file mode 100644 index 00000000..75be9c6d --- /dev/null +++ b/src/observability/types/scan-observability.types.ts @@ -0,0 +1,78 @@ +/** + * Types for Scan Observability, Throughput Dashboard, Saturation Metrics, + * Stage Duration Histograms, and Failure Taxonomy (#1259, #1260, #1261, #1264). + */ + +export type ScanStage = + | 'cloning' + | 'ast_generation' + | 'rule_evaluation' + | 'fingerprinting' + | 'reporting' + | 'persisting'; + +export interface StageHistogramBucket { + le: number; // upper bound in milliseconds + count: number; +} + +export interface StageDurationHistogram { + stage: ScanStage; + count: number; + sumMs: number; + buckets: StageHistogramBucket[]; +} + +export interface QueuePressureMetrics { + activeWorkers: number; + maxWorkers: number; + queuedJobs: number; + saturationRatio: number; // activeWorkers / maxWorkers + isSaturated: boolean; + averageWaitTimeMs: number; + timestamp: number; +} + +export type ScanFailureCategory = + | 'SYNTAX_ERROR' + | 'TIMEOUT' + | 'RESOURCE_EXHAUSTION' + | 'RULE_EXECUTION_FAILURE' + | 'DEPENDENCY_RESOLVE_ERROR' + | 'NETWORK_IO_ERROR' + | 'INTERNAL_ERROR'; + +export interface StructuredScanFailure { + failureId: string; + scanId: string; + tenantId: string; + category: ScanFailureCategory; + errorCode: string; + message: string; + stage: ScanStage; + retryable: boolean; + timestamp: number; + context?: Record; +} + +export interface ScanThroughputRecord { + scanId: string; + tenantId: string; + repository: string; + filesScanned: number; + durationMs: number; + timestamp: number; + success: boolean; +} + +export interface ThroughputDashboardSummary { + windowMs: number; + totalScans: number; + successfulScans: number; + failedScans: number; + successRate: number; + totalFilesScanned: number; + scansPerMinute: number; + filesPerSecond: number; + avgDurationMs: number; +}