From 4b05e31cef1f1fc4eef148ba1988e1489eca63eb Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 13:10:12 -0700 Subject: [PATCH 01/19] feat(advisories): add date-range filtering for published/updated - Add parseDateFilter + filterByDateRange to LocalRepositoryDataSource: single day (YYYY-MM-DD) and inclusive range (YYYY-MM-DD..YYYY-MM-DD) with validation for malformed/reversed ranges and array inputs - Replace naive '>=' comparison that only supported open-ended dates - Improve list_advisories schema descriptions (date format, examples, defaults) - Add 14 unit tests covering parse/filter edge cases Cleanly re-applies the feature from #25 onto current main (post-#76); supersedes that stale branch. --- src/datasources/local-repository.ts | 64 +++++++++- src/tools/advisories.ts | 16 +-- test/unit/local-repository.test.ts | 180 ++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 10 deletions(-) create mode 100644 test/unit/local-repository.test.ts diff --git a/src/datasources/local-repository.ts b/src/datasources/local-repository.ts index 8988dde..0ef5294 100644 --- a/src/datasources/local-repository.ts +++ b/src/datasources/local-repository.ts @@ -301,6 +301,66 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { }; } + /** + * Parse date filter string and return start/end dates. + * Supports: "2026-01-27" (single day) or "2026-01-01..2026-01-31" (range). + * + * Throws an Error with a descriptive message when the input is malformed: + * - more than two parts in a range + * - missing start or end in a range + * - invalid calendar date (e.g. "2026-13-45") + * - reversed range (start > end) + */ + private parseDateFilter(dateStr: string): { start: string; end: string } { + // Defense-in-depth: ensure dateStr is a string (HTTP query params can be arrays) + const str = Array.isArray(dateStr) ? String(dateStr[0]) : String(dateStr); + + if (str.includes('..')) { + const parts = str.split('..'); + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Error( + `Invalid date range "${str}". Expected "YYYY-MM-DD..YYYY-MM-DD".` + ); + } + const [start, end] = parts; + const startDate = new Date(start + 'T00:00:00Z'); + const endDate = new Date(end + 'T00:00:00Z'); + if (isNaN(startDate.getTime()) || isNaN(endDate.getTime())) { + throw new Error( + `Invalid date in range "${str}". Each side must be YYYY-MM-DD.` + ); + } + if (startDate.getTime() > endDate.getTime()) { + throw new Error( + `Invalid date range "${str}": start date must be less than or equal to end date.` + ); + } + // End date: include full day by using next day midnight (end is exclusive) + endDate.setUTCDate(endDate.getUTCDate() + 1); + return { start: startDate.toISOString(), end: endDate.toISOString() }; + } + + // Single date: filter for that specific day + const startDate = new Date(str + 'T00:00:00Z'); + const endDate = new Date(str + 'T00:00:00Z'); + if (isNaN(startDate.getTime()) || isNaN(endDate.getTime())) { + throw new Error(`Invalid date "${str}". Expected YYYY-MM-DD.`); + } + endDate.setUTCDate(endDate.getUTCDate() + 1); + return { start: startDate.toISOString(), end: endDate.toISOString() }; + } + + /** + * Filter advisories by date range + */ + private filterByDateRange(advisories: Advisory[], field: 'published_at' | 'updated_at', dateStr: string): Advisory[] { + const { start, end } = this.parseDateFilter(dateStr); + return advisories.filter(a => { + const date = a[field]; + return date >= start && date < end; + }); + } + /** * List advisories with optional filtering */ @@ -349,11 +409,11 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { } if (options.published) { - results = results.filter(a => a.published_at >= options.published!); + results = this.filterByDateRange(results, 'published_at', options.published); } if (options.updated) { - results = results.filter(a => a.updated_at >= options.updated!); + results = this.filterByDateRange(results, 'updated_at', options.updated); } // Sort results diff --git a/src/tools/advisories.ts b/src/tools/advisories.ts index f8b7a74..822cf6a 100644 --- a/src/tools/advisories.ts +++ b/src/tools/advisories.ts @@ -44,18 +44,18 @@ function buildQueryString(params: Record): string { * List global security advisories from local database */ export const listAdvisoriesSchema = z.object({ - ghsa_id: z.string().optional().describe('GHSA identifier'), - cve_id: z.string().optional().describe('CVE identifier'), + ghsa_id: z.string().optional().describe('GHSA identifier (e.g., "GHSA-xxxx-xxxx-xxxx")'), + cve_id: z.string().optional().describe('CVE identifier (e.g., "CVE-2026-12345")'), ecosystem: z.enum(['rubygems', 'npm', 'pip', 'maven', 'nuget', 'composer', 'go', 'rust', 'erlang', 'actions', 'pub', 'other', 'swift']).optional().describe('Package ecosystem'), severity: z.enum(['low', 'medium', 'high', 'critical', 'unknown']).optional().describe('Severity level'), cwes: z.string().optional().describe('Comma-separated CWE identifiers (e.g., "79,284,22")'), is_withdrawn: z.boolean().optional().describe('Filter withdrawn advisories'), - affects: z.string().optional().describe('Package name filter'), - published: z.string().optional().describe('Published date or range'), - updated: z.string().optional().describe('Updated date or range'), - per_page: z.number().min(1).max(100).optional().describe('Results per page (max 100)'), - direction: z.enum(['asc', 'desc']).optional().describe('Sort direction'), - sort: z.enum(['updated', 'published']).optional().describe('Sort field') + affects: z.string().optional().describe('Package name filter (partial match, e.g., "express" matches "express-session")'), + published: z.string().optional().describe('Filter by published date in YYYY-MM-DD format. Single date returns that day only. Range format: "2026-01-01..2026-01-31" returns inclusive range. Examples: "2026-01-27" (single day), "2026-01-01..2026-01-31" (January 2026)'), + updated: z.string().optional().describe('Filter by updated date in YYYY-MM-DD format. Single date returns that day only. Range format: "2026-01-01..2026-01-31" returns inclusive range'), + per_page: z.number().min(1).max(100).optional().describe('Results per page (default: 30, max: 100)'), + direction: z.enum(['asc', 'desc']).optional().describe('Sort direction (default: desc, newest first)'), + sort: z.enum(['updated', 'published']).optional().describe('Sort field (default: published)') }); export async function listAdvisories(params: unknown): Promise { diff --git a/test/unit/local-repository.test.ts b/test/unit/local-repository.test.ts new file mode 100644 index 0000000..c91f027 --- /dev/null +++ b/test/unit/local-repository.test.ts @@ -0,0 +1,180 @@ +/** + * Unit tests for LocalRepositoryDataSource date filtering logic. + * + * Covers `parseDateFilter` and `filterByDateRange` edge cases that the + * E2E suite cannot reach reliably (invalid input, reversed ranges, etc.). + */ + +import { describe, it, expect, beforeEach } from "vitest"; +import { LocalRepositoryDataSource } from "../../src/datasources/local-repository.js"; +import type { Advisory } from "../../src/types/data-source.js"; + +// Type alias for accessing the private members under test without changing +// the production class's public API. +type DateFilterInternals = { + parseDateFilter(dateStr: string): { start: string; end: string }; + filterByDateRange( + advisories: Advisory[], + field: "published_at" | "updated_at", + dateStr: string + ): Advisory[]; +}; + +describe("LocalRepositoryDataSource - parseDateFilter", () => { + let internals: DateFilterInternals; + + beforeEach(() => { + const ds = new LocalRepositoryDataSource("/nonexistent/repo"); + internals = ds as unknown as DateFilterInternals; + }); + + describe("single date", () => { + it("returns full-day window for a valid YYYY-MM-DD", () => { + const { start, end } = internals.parseDateFilter("2026-01-27"); + expect(start).toBe("2026-01-27T00:00:00.000Z"); + expect(end).toBe("2026-01-28T00:00:00.000Z"); + }); + + it("throws on invalid month", () => { + expect(() => internals.parseDateFilter("2026-13-01")).toThrowError( + /Invalid date "2026-13-01"/ + ); + }); + + it("throws on completely malformed input", () => { + expect(() => internals.parseDateFilter("not-a-date")).toThrowError( + /Invalid date "not-a-date"/ + ); + }); + }); + + describe("date range", () => { + it("returns inclusive range for valid YYYY-MM-DD..YYYY-MM-DD", () => { + const { start, end } = internals.parseDateFilter( + "2026-01-01..2026-01-31" + ); + expect(start).toBe("2026-01-01T00:00:00.000Z"); + // End is exclusive (next day midnight) so January 31 is fully included. + expect(end).toBe("2026-02-01T00:00:00.000Z"); + }); + + it("accepts equal start and end (single-day range)", () => { + const { start, end } = internals.parseDateFilter( + "2026-01-15..2026-01-15" + ); + expect(start).toBe("2026-01-15T00:00:00.000Z"); + expect(end).toBe("2026-01-16T00:00:00.000Z"); + }); + + it("throws when range has more than two parts", () => { + expect(() => + internals.parseDateFilter("2026-01-01..2026-02-01..2026-03-01") + ).toThrowError(/Invalid date range/); + }); + + it("throws when range is missing the end side", () => { + expect(() => internals.parseDateFilter("2026-01-01..")).toThrowError( + /Invalid date range/ + ); + }); + + it("throws when range is missing the start side", () => { + expect(() => internals.parseDateFilter("..2026-01-31")).toThrowError( + /Invalid date range/ + ); + }); + + it("throws when start side is not a real date", () => { + expect(() => + internals.parseDateFilter("2026-13-45..2026-12-31") + ).toThrowError(/Invalid date in range/); + }); + + it("throws when end side is not a real date", () => { + expect(() => + internals.parseDateFilter("2026-01-01..invalid") + ).toThrowError(/Invalid date in range/); + }); + + it("throws when start is after end (reversed range)", () => { + expect(() => + internals.parseDateFilter("2026-12-31..2026-01-01") + ).toThrowError(/start date must be less than or equal to end date/); + }); + }); +}); + +describe("LocalRepositoryDataSource - filterByDateRange", () => { + let internals: DateFilterInternals; + + beforeEach(() => { + const ds = new LocalRepositoryDataSource("/nonexistent/repo"); + internals = ds as unknown as DateFilterInternals; + }); + + function makeAdvisory(id: string, publishedIso: string): Advisory { + return { + ghsa_id: id, + cve_id: null, + url: "", + html_url: "", + summary: "", + description: "", + type: "reviewed", + severity: "high", + repository_advisory_url: null, + source_code_location: null, + identifiers: [], + references: [], + published_at: publishedIso, + updated_at: publishedIso, + github_reviewed_at: publishedIso, + nvd_published_at: null, + withdrawn_at: null, + vulnerabilities: [], + cvss: { vector_string: null, score: null }, + cvss_severities: { + cvss_v3: { vector_string: null, score: null }, + cvss_v4: { vector_string: null, score: null }, + }, + epss: undefined, + cwes: [], + credits: [], + } as unknown as Advisory; + } + + it("returns only advisories within an inclusive single-day window", () => { + const advisories = [ + makeAdvisory("A", "2026-01-26T23:59:59.000Z"), + makeAdvisory("B", "2026-01-27T00:00:00.000Z"), + makeAdvisory("C", "2026-01-27T12:30:00.000Z"), + makeAdvisory("D", "2026-01-28T00:00:00.000Z"), + ]; + const filtered = internals.filterByDateRange( + advisories, + "published_at", + "2026-01-27" + ); + expect(filtered.map((a) => a.ghsa_id)).toEqual(["B", "C"]); + }); + + it("includes the end date in a YYYY-MM-DD..YYYY-MM-DD range", () => { + const advisories = [ + makeAdvisory("A", "2026-01-31T23:00:00.000Z"), + makeAdvisory("B", "2026-02-01T00:00:00.000Z"), + ]; + const filtered = internals.filterByDateRange( + advisories, + "published_at", + "2026-01-01..2026-01-31" + ); + expect(filtered.map((a) => a.ghsa_id)).toEqual(["A"]); + }); + + it("propagates errors for invalid input", () => { + const advisories = [makeAdvisory("A", "2026-01-15T00:00:00.000Z")]; + expect(() => + internals.filterByDateRange(advisories, "published_at", "bad-date") + ).toThrowError(/Invalid date "bad-date"/); + }); +}); From ce5429f4b6ec9a4fc4affaadec45cae44487f1e6 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 15:21:41 -0700 Subject: [PATCH 02/19] fix(datasource): map ecosystem enum to OSV names so filtering works (#78) list_advisories/search ecosystem filter used exact string match against OSV data, so only 'npm' matched; composer/pip/maven/rust/etc. silently returned 0. - Add ECOSYSTEM_ALIASES (GitHub enum -> OSV name) + case-insensitive ecosystemMatches() - Apply at both filter sites (listAdvisories + filterResults/search) - Add unit tests for all 12 ecosystems; make e2e assertions real regression guards (drop vacuous length>0 guard, use ecosystemMatches instead of fragile [0]===name) Fixes #78 --- src/datasources/local-repository.ts | 31 +++++++++++++++++++-- test/e2e/mcp-server.test.ts | 22 +++++++++------ test/unit/ecosystem-matching.test.ts | 41 ++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 test/unit/ecosystem-matching.test.ts diff --git a/src/datasources/local-repository.ts b/src/datasources/local-repository.ts index 8988dde..c08ace2 100644 --- a/src/datasources/local-repository.ts +++ b/src/datasources/local-repository.ts @@ -63,6 +63,33 @@ interface OSVAdvisory { }>; } +/** + * Maps the GitHub-style ecosystem names accepted by the tool schema to the OSV + * ecosystem names actually stored in advisory-database (differs in name/casing). + */ +const ECOSYSTEM_ALIASES: Record = { + npm: 'npm', + pip: 'PyPI', + maven: 'Maven', + nuget: 'NuGet', + rubygems: 'RubyGems', + composer: 'Packagist', + go: 'Go', + rust: 'crates.io', + erlang: 'Hex', + pub: 'Pub', + swift: 'SwiftURL', + actions: 'GitHub Actions', +}; + +/** + * Case-insensitive ecosystem match that also accepts the OSV name directly. + */ +export function ecosystemMatches(packageEcosystem: string, requested: string): boolean { + const canonical = ECOSYSTEM_ALIASES[requested.toLowerCase()] ?? requested; + return packageEcosystem.toLowerCase() === canonical.toLowerCase(); +} + /** * Data source that reads from local cloned github/advisory-database repository */ @@ -320,7 +347,7 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { if (options.ecosystem) { results = results.filter(a => - a.vulnerabilities.some(v => v.package.ecosystem === options.ecosystem) + a.vulnerabilities.some(v => ecosystemMatches(v.package.ecosystem, options.ecosystem!)) ); } @@ -411,7 +438,7 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { // Apply all filters from listAdvisories if (options.ecosystem) { results = results.filter(a => - a.vulnerabilities.some(v => v.package.ecosystem === options.ecosystem) + a.vulnerabilities.some(v => ecosystemMatches(v.package.ecosystem, options.ecosystem!)) ); } diff --git a/test/e2e/mcp-server.test.ts b/test/e2e/mcp-server.test.ts index cf0097a..58609ed 100644 --- a/test/e2e/mcp-server.test.ts +++ b/test/e2e/mcp-server.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { ChildProcess } from "child_process"; +import { ecosystemMatches } from "../../src/datasources/local-repository.js"; import { startMCPServer, stopMCPServer, @@ -114,7 +115,9 @@ describe("MCP Advisory Server E2E Tests", () => { expect(advisory).toHaveProperty("summary"); expect(advisory).toHaveProperty("severity"); expect(advisory.affected_packages).toBeInstanceOf(Array); - expect(advisory.affected_packages[0].ecosystem).toBe("npm"); + expect( + advisory.affected_packages.some((p: any) => ecosystemMatches(p.ecosystem, "npm")) + ).toBe(true); }); it("should filter by severity", async () => { @@ -145,20 +148,21 @@ describe("MCP Advisory Server E2E Tests", () => { }); it("should list multiple ecosystems", async () => { - for (const ecosystem of ["npm", "pip", "maven", "go"]) { + // Regression guard: ecosystem enum values must map to OSV names in the data, + // so each ecosystem must return results (was silently 0 for all but npm). + for (const ecosystem of ["npm", "pip", "maven", "go", "composer"]) { const response = await callMCPTool(baseUrl, sessionId, "list_advisories", { ecosystem, per_page: 1, }); const content = JSON.parse(response.result.content[0].text); - if (content.advisories.length > 0) { - expect( - content.advisories[0].affected_packages.some( - (p: any) => p.ecosystem === ecosystem - ) - ).toBe(true); - } + expect(content.advisories.length).toBeGreaterThan(0); + expect( + content.advisories[0].affected_packages.some( + (p: any) => ecosystemMatches(p.ecosystem, ecosystem) + ) + ).toBe(true); } }); }); diff --git a/test/unit/ecosystem-matching.test.ts b/test/unit/ecosystem-matching.test.ts new file mode 100644 index 0000000..3501a67 --- /dev/null +++ b/test/unit/ecosystem-matching.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect } from "vitest"; +import { ecosystemMatches } from "../../src/datasources/local-repository.js"; + +describe("ecosystemMatches", () => { + // GitHub tool-schema enum value -> OSV ecosystem name stored in advisory-database + const cases: Array<[string, string]> = [ + ["npm", "npm"], + ["pip", "PyPI"], + ["maven", "Maven"], + ["nuget", "NuGet"], + ["rubygems", "RubyGems"], + ["composer", "Packagist"], + ["go", "Go"], + ["rust", "crates.io"], + ["erlang", "Hex"], + ["pub", "Pub"], + ["swift", "SwiftURL"], + ["actions", "GitHub Actions"], + ]; + + it.each(cases)( + "matches GitHub enum '%s' against OSV name '%s'", + (enumValue, osvName) => { + expect(ecosystemMatches(osvName, enumValue)).toBe(true); + } + ); + + it("accepts the OSV name passed directly", () => { + expect(ecosystemMatches("Packagist", "Packagist")).toBe(true); + expect(ecosystemMatches("PyPI", "pypi")).toBe(true); + }); + + it("is case-insensitive on the package ecosystem", () => { + expect(ecosystemMatches("packagist", "composer")).toBe(true); + }); + + it("does not match unrelated ecosystems", () => { + expect(ecosystemMatches("PyPI", "composer")).toBe(false); + expect(ecosystemMatches("npm", "pip")).toBe(false); + }); +}); From a3370338ff800f8a79958acf40c1e24e6927809e Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 17:26:55 -0700 Subject: [PATCH 03/19] fix(datasource): normalize CWE filter input (#80) cwes filter did options.cwes.includes(cwe.cwe_id) i.e. checked if bare input like '89' contains 'CWE-89' - always false. Documented bare-number form never matched. - Add normalizeCwe() + cweFilterMatches(): accept bare (89) or prefixed (CWE-89), comma-separated and/or array, case-insensitive; match if any requested CWE is present - Add unit tests (test/unit/cwe-filter.test.ts) Fixes #80 --- src/datasources/local-repository.ts | 27 +++++++++++++++++++- test/unit/cwe-filter.test.ts | 39 +++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 test/unit/cwe-filter.test.ts diff --git a/src/datasources/local-repository.ts b/src/datasources/local-repository.ts index c08ace2..b98d9e3 100644 --- a/src/datasources/local-repository.ts +++ b/src/datasources/local-repository.ts @@ -90,6 +90,31 @@ export function ecosystemMatches(packageEcosystem: string, requested: string): b return packageEcosystem.toLowerCase() === canonical.toLowerCase(); } +/** + * Normalize a CWE token to canonical `CWE-` (uppercase), accepting bare + * numbers (`89`) or already-prefixed ids (`CWE-89`). + */ +export function normalizeCwe(token: string): string { + const t = String(token).trim(); + return (/^cwe-/i.test(t) ? t : `CWE-${t}`).toUpperCase(); +} + +/** + * True if an advisory's CWE ids intersect the requested filter. Requested + * values may be an array and/or comma-separated, bare or prefixed. + */ +export function cweFilterMatches(advisoryCweIds: string[], requested: string[]): boolean { + const wanted = new Set( + requested + .flatMap(c => String(c).split(',')) + .map(s => s.trim()) + .filter(Boolean) + .map(normalizeCwe) + ); + if (wanted.size === 0) return true; + return advisoryCweIds.some(id => wanted.has(normalizeCwe(id))); +} + /** * Data source that reads from local cloned github/advisory-database repository */ @@ -359,7 +384,7 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { if (options.cwes && options.cwes.length > 0) { results = results.filter(a => - a.cwes.some(cwe => options.cwes!.includes(cwe.cwe_id)) + cweFilterMatches(a.cwes.map(cwe => cwe.cwe_id), options.cwes!) ); } diff --git a/test/unit/cwe-filter.test.ts b/test/unit/cwe-filter.test.ts new file mode 100644 index 0000000..7e50e73 --- /dev/null +++ b/test/unit/cwe-filter.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from "vitest"; +import { normalizeCwe, cweFilterMatches } from "../../src/datasources/local-repository.js"; + +describe("normalizeCwe", () => { + it("prefixes bare numbers", () => { + expect(normalizeCwe("89")).toBe("CWE-89"); + }); + it("keeps and upper-cases prefixed ids", () => { + expect(normalizeCwe("cwe-89")).toBe("CWE-89"); + expect(normalizeCwe("CWE-89")).toBe("CWE-89"); + }); + it("trims whitespace", () => { + expect(normalizeCwe(" 79 ")).toBe("CWE-79"); + }); +}); + +describe("cweFilterMatches", () => { + const adv = ["CWE-89", "CWE-943"]; + + it("matches a bare number (the documented input form)", () => { + expect(cweFilterMatches(adv, ["89"])).toBe(true); + }); + it("matches a prefixed id", () => { + expect(cweFilterMatches(adv, ["CWE-943"])).toBe(true); + }); + it("matches a comma-separated single element", () => { + expect(cweFilterMatches(adv, ["79,89"])).toBe(true); + }); + it("matches across a multi-element array", () => { + expect(cweFilterMatches(adv, ["79", "89"])).toBe(true); + }); + it("does not match unrelated CWEs", () => { + expect(cweFilterMatches(adv, ["79"])).toBe(false); + expect(cweFilterMatches(adv, ["22,306"])).toBe(false); + }); + it("empty filter matches everything", () => { + expect(cweFilterMatches(adv, [])).toBe(true); + }); +}); From aa01e27d4af3b87e4721f8f4e6c6f495bbe3d087 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 17:32:29 -0700 Subject: [PATCH 04/19] fix(tools): derive local API URL from ADVISORY_API_PORT (#81) The MCP tools read ADVISORY_API_BASE (hardcoded :18005) while the server binds ADVISORY_API_PORT, so a custom API port broke every tool with 'fetch failed', and two servers collided on 18005 (serving each other's data - the source of the E2E flakiness). - Derive base URL from ADVISORY_API_HOST/PORT; keep ADVISORY_API_BASE as override - CI E2E now runs on a non-default API port (18055) to guard against regressions Fixes #81 --- .github/workflows/build.yml | 5 +++-- src/tools/advisories.ts | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 564528a..48ac640 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,5 +65,6 @@ jobs: run: npm run test:e2e env: ADVISORY_REPO_PATH: ./external/advisory-database - MCP_PORT: '18006' - ADVISORY_API_PORT: '18005' \ No newline at end of file + # Non-default API port: guards against the tool hardcoding 18005 (#81). + MCP_PORT: '18056' + ADVISORY_API_PORT: '18055' \ No newline at end of file diff --git a/src/tools/advisories.ts b/src/tools/advisories.ts index f8b7a74..69c5c37 100644 --- a/src/tools/advisories.ts +++ b/src/tools/advisories.ts @@ -5,10 +5,13 @@ import { createLogger } from '../logger.js'; const logger = createLogger('Tools'); /** - * Local advisory server configuration - * Set via environment variable or default to local instance + * Local advisory server configuration. + * Derived from the same ADVISORY_API_HOST/PORT the server binds, so a custom + * port reaches the right instance; ADVISORY_API_BASE overrides the full URL. */ -const LOCAL_API_BASE = process.env.ADVISORY_API_BASE || 'http://localhost:18005'; +const LOCAL_API_BASE = + process.env.ADVISORY_API_BASE || + `http://${process.env.ADVISORY_API_HOST || '127.0.0.1'}:${process.env.ADVISORY_API_PORT || '18005'}`; /** * Fetch data from local advisory API From 34435ed638c606983927ef7dd2aebffdc8325f5b Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 17:34:07 -0700 Subject: [PATCH 05/19] chore(mcp): default advisory server to ADVISORY_API_PORT=18025 Distinct from the test defaults (18005/18006) so the dev MCP server and the test suite can run simultaneously without colliding. Relies on the tool port fix in this PR. --- .vscode/mcp.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/mcp.json b/.vscode/mcp.json index fe4f10a..484e5b3 100644 --- a/.vscode/mcp.json +++ b/.vscode/mcp.json @@ -7,7 +7,8 @@ ], "type": "stdio", "env": { - "ADVISORY_REPO_PATH": "${workspaceFolder}/external/advisory-database" + "ADVISORY_REPO_PATH": "${workspaceFolder}/external/advisory-database", + "ADVISORY_API_PORT": "18025" } } } From d5e6f77a15316655c1fa48903a73a909cfa9853f Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 19:54:17 -0700 Subject: [PATCH 06/19] feat(local): reviewed/unreviewed tier filter + web_app_only (session prototype) --- src/datasources/local-repository.ts | 52 +++++++++++++++++++++++++++-- src/local-server.ts | 2 ++ src/tools/advisories.ts | 4 ++- src/types/data-source.ts | 4 +++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/datasources/local-repository.ts b/src/datasources/local-repository.ts index 2e15cbc..16092c5 100644 --- a/src/datasources/local-repository.ts +++ b/src/datasources/local-repository.ts @@ -115,6 +115,38 @@ export function cweFilterMatches(advisoryCweIds: string[], requested: string[]): return advisoryCweIds.some(id => wanted.has(normalizeCwe(id))); } +/** + * Heuristic set of CWEs that denote web-application vulnerability classes + * (OWASP-aligned: injection, XSS, SSRF/CSRF, path traversal, auth/z, session, + * deserialization, request smuggling, etc.). Used to categorize web-app vs + * cloud/local issues. Extend as needed. + */ +const WEB_APP_CWES: Set = new Set( + [ + // Injection + 79, 89, 77, 78, 90, 91, 93, 94, 95, 113, 116, 564, 917, 943, 1336, 1333, + // Path / file + 22, 23, 36, 73, 98, 434, 552, 610, + // Request forgery / redirect / UI redress + 352, 601, 918, 1021, + // XML + 611, 776, 827, + // AuthN / AuthZ / session / secrets-over-web + 287, 306, 384, 522, 613, 620, 639, 640, 862, 863, 1275, + // Info exposure over web + 200, 209, 532, 548, + // Deserialization / parsing / request handling + 347, 345, 444, 502, + ].map(n => `CWE-${n}`) +); + +/** + * True if the advisory looks like a web-application vulnerability (by CWE). + */ +export function isWebAppAdvisory(cweIds: string[]): boolean { + return cweIds.some(id => WEB_APP_CWES.has(normalizeCwe(id))); +} + /** * Data source that reads from local cloned github/advisory-database repository */ @@ -226,7 +258,12 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { const reviewedPath = join(this.repoPath, 'advisories', 'github-reviewed'); await this.indexDirectory(reviewedPath); - + + // Opt-in: also index the (much larger) unreviewed tier for "all" searches. + if (process.env.ADVISORY_INCLUDE_UNREVIEWED === 'true') { + await this.indexDirectory(join(this.repoPath, 'advisories', 'unreviewed')); + } + this.indexBuilt = true; } @@ -331,7 +368,7 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { repository_advisory_url: null, summary: osv.summary || osv.details.split('\n')[0], description: osv.details, - type: 'reviewed', + type: osv.database_specific.github_reviewed ? 'reviewed' : 'unreviewed', severity: osv.database_specific.severity.toLowerCase(), source_code_location: null, identifiers: [ @@ -448,6 +485,17 @@ export class LocalRepositoryDataSource implements IAdvisoryDataSource { ); } + // Review tier: default to reviewed-only for parity with prior behavior. + const tier = options.type || 'reviewed'; + if (tier !== 'all') { + results = results.filter(a => a.type === tier); + } + + // Web-application focus: keep only advisories whose CWEs are web-app classes. + if (options.web_app_only) { + results = results.filter(a => isWebAppAdvisory(a.cwes.map(c => c.cwe_id))); + } + if (options.is_withdrawn !== undefined) { results = results.filter(a => options.is_withdrawn ? a.withdrawn_at !== null : a.withdrawn_at === null diff --git a/src/local-server.ts b/src/local-server.ts index caa02a7..14a1349 100644 --- a/src/local-server.ts +++ b/src/local-server.ts @@ -51,6 +51,8 @@ export async function createLocalAdvisoryServer(config: LocalServerConfig) { published: req.query.published as string, updated: req.query.updated as string, modified: req.query.modified as string, + type: (req.query.type as 'reviewed' | 'unreviewed' | 'all') || undefined, + web_app_only: req.query.web_app_only === 'true' ? true : undefined, per_page: req.query.per_page ? parseInt(req.query.per_page as string) : undefined, page: req.query.page ? parseInt(req.query.page as string) : undefined, sort: (req.query.sort as 'published' | 'updated') || 'published', diff --git a/src/tools/advisories.ts b/src/tools/advisories.ts index 3207204..a0f9e5b 100644 --- a/src/tools/advisories.ts +++ b/src/tools/advisories.ts @@ -58,7 +58,9 @@ export const listAdvisoriesSchema = z.object({ updated: z.string().optional().describe('Filter by updated date in YYYY-MM-DD format. Single date returns that day only. Range format: "2026-01-01..2026-01-31" returns inclusive range'), per_page: z.number().min(1).max(100).optional().describe('Results per page (default: 30, max: 100)'), direction: z.enum(['asc', 'desc']).optional().describe('Sort direction (default: desc, newest first)'), - sort: z.enum(['updated', 'published']).optional().describe('Sort field (default: published)') + sort: z.enum(['updated', 'published']).optional().describe('Sort field (default: published)'), + type: z.enum(['reviewed', 'unreviewed', 'all']).optional().describe('Review tier: "reviewed" (default, curated ~35k), "unreviewed" (raw, ~335k), or "all". Non-reviewed requires the server to run with ADVISORY_INCLUDE_UNREVIEWED=true.'), + web_app_only: z.boolean().optional().describe('Keep only web-application vulnerability classes by CWE (injection, XSS, SSRF/CSRF, path traversal, auth/z, session, deserialization, request smuggling)') }); export async function listAdvisories(params: unknown): Promise { diff --git a/src/types/data-source.ts b/src/types/data-source.ts index d2362c6..c4bd356 100644 --- a/src/types/data-source.ts +++ b/src/types/data-source.ts @@ -14,6 +14,10 @@ export interface AdvisoryListOptions { published?: string; updated?: string; modified?: string; + /** Which review tier to include (default 'reviewed'). 'all'/'unreviewed' require ADVISORY_INCLUDE_UNREVIEWED=true at index time. */ + type?: 'reviewed' | 'unreviewed' | 'all'; + /** Keep only advisories whose CWEs map to web-application vulnerability classes. */ + web_app_only?: boolean; epss_percentage?: string; epss_percentile?: string; before?: string; From f9962871aba5633f547f4044b4879a15e9007fc0 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 20:10:39 -0700 Subject: [PATCH 07/19] proto(semantic): local hybrid search (embeddings + BM25 + RRF + temporal rerank) Local-only, advisory-specific prototype on branch proto/semantic-search. No external engine/service. - Local ONNX embeddings via @huggingface/transformers (MiniLM 384-dim), offline (reuse a cached model dir) - Compact Okapi BM25 with identifier-preserving tokenizer - RRF fusion + field-aware rerank (exact GHSA/CVE/package/CWE/phrase boosts) - Temporal-aware rerank: parse a period from the query, run recall on the residual text, boost by publish-date proximity (in-window 1.0, exp decay half-life 45d) - File-backed index (embeddings.bin/bm25.json/docs.json/meta.json), CLI build+query. Not wired into MCP tools yet. --- .gitignore | 4 + package-lock.json | 865 +++++++++++++++++++++++++++++++++++- package.json | 5 +- src/semantic/README.md | 70 +++ src/semantic/bm25.ts | 94 ++++ src/semantic/build-index.ts | 57 +++ src/semantic/document.ts | 59 +++ src/semantic/embeddings.ts | 57 +++ src/semantic/hybrid.ts | 111 +++++ src/semantic/query.ts | 34 ++ src/semantic/store.ts | 61 +++ src/semantic/temporal.ts | 101 +++++ 12 files changed, 1515 insertions(+), 3 deletions(-) create mode 100644 src/semantic/README.md create mode 100644 src/semantic/bm25.ts create mode 100644 src/semantic/build-index.ts create mode 100644 src/semantic/document.ts create mode 100644 src/semantic/embeddings.ts create mode 100644 src/semantic/hybrid.ts create mode 100644 src/semantic/query.ts create mode 100644 src/semantic/store.ts create mode 100644 src/semantic/temporal.ts diff --git a/.gitignore b/.gitignore index 5707b63..5be087e 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,7 @@ Thumbs.db # External advisory database (git submodule/clone) external/ + +# Semantic-search prototype: local index + downloaded model weights +.semantic-index/ +models/ diff --git a/package-lock.json b/package-lock.json index 5a0e2ec..efc0d02 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@huggingface/transformers": "^3.8.1", "@modelcontextprotocol/sdk": "^1.30.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/api-logs": "^0.222.0", @@ -363,6 +364,16 @@ "kuler": "^2.0.0" } }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha1-hCV647BTHrKuwf+iPXBwDaAHupU=", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", @@ -848,6 +859,492 @@ "hono": "^4" } }, + "node_modules/@huggingface/jinja": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.10.tgz", + "integrity": "sha1-UeRpuc9DGYzMj3qKZcSNmXgRng8=", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/transformers": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.8.1.tgz", + "integrity": "sha1-MX2gA4ZTIjlnlhcyI+6q8PlyPwo=", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.3", + "onnxruntime-node": "1.21.0", + "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", + "sharp": "^0.34.1" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha1-sMLC+mYa33Xv/WtJZEl82AAQu50=", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha1-bgcy3K3hJrZnCveqFwYLkmg16oY=", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha1-Gbwd1uum1alig0mLnJ9AEYDunHs=", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha1-KJTAy4fUInbDiJlC6OLbUXpJLEM=", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha1-5jaB9FOalK+c0XJG7YiBc0OG+Mw=", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha1-uSYN0evm+eO9vL3KydKsEl81hS0=", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha1-sbKIs2hks7zlRa2R+m2tzxpK0xg=", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha1-S4Ps8qgpBXIis4hIx7Ai57TQeqc=", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha1-iAtGeACeWiCArxkjMrALCq+KSN4=", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha1-dPNDyOEPrYIbOPdc7TBIiTncWew=", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha1-30GD6L2EEPfWG2aFmjXt6rClMc4=", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha1-yNa0ghHfZxN1QQB+6NG3sfjKjgY=", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha1-vhHHW+5bCAy+4xoVOod5RI+Rn3U=", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha1-X7DDaV3RJSLTnD/3pryBZGF4Cg0=", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha1-eqd2TvnAAfFeYQVG1C/OVpEXkMw=", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha1-nCE6gVIKIMr2aXjz1MB0Vv8uCBM=", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha1-zdKBgndOrb4E9iZ1oWqrvMuDP2A=", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha1-k+rGAbnzKbsnkX4OGQmMci1jDfc=", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha1-VavHzXVP/KUAK2wrcZq9/IRoGag=", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha1-1lFe6XG7YvcwAaSCm52GWhG3cIY=", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha1-2Xl4rsfFIS+ZlxTy9bc2RX4S7p8=", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha1-LxWAOqYm+MWd18nQu8dm8atSz6A=", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha1-Nwbp46w1/d/ByH+U6Enxt1MHzgo=", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha1-C3EWZZmwSeAy8IX7kmPgL05HiN4=", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha1-qB/7AOaSZ80KHWJurtuKhDCysvg=", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -865,6 +1362,18 @@ "node": ">=12" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha1-LVmuOrSzj7QnC/oj0w+OLobH/jI=", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", @@ -3226,6 +3735,13 @@ "url": "https://opencollective.com/express" } }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha1-nlKUr06YMUSUy7F5efpUyhWfEWs=", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", @@ -3306,6 +3822,15 @@ "node": ">=18" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha1-mFXmTs0kCpzEJnzopKpdJKHaFeQ=", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/cjs-module-lexer": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.1.tgz", @@ -3597,6 +4122,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha1-iU3BQbt9MGCuQ2b2oBB+aPvkjF4=", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/define-lazy-prop": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", @@ -3610,6 +4152,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha1-EHgcxhbrlRqAoDS6/Kpzd/avK2w=", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -3619,6 +4178,21 @@ "node": ">= 0.8" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha1-aJxdzcGQDvVYOky59te0c3QgdK0=", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha1-yccHdaScPQO8LAbZpzvlUPl4+LE=", + "license": "MIT" + }, "node_modules/dotenv": { "version": "17.4.2", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", @@ -3725,6 +4299,12 @@ "node": ">= 0.4" } }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha1-njr0B0Wd7tR+mpH5uIWoTrBcVh0=", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -3782,6 +4362,18 @@ "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=", "license": "MIT" }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha1-FLqDpdNz49MR5a/KKc9b+tllvzQ=", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -3990,6 +4582,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha1-NGgRVX/pMSq1ZHU155PHYenIHrE=", + "license": "Apache-2.0" + }, "node_modules/fn.name": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", @@ -4167,6 +4765,39 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha1-rnzTG9NYO5PFoWQ3oa/ifMM6GrY=", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha1-dDDtOpddl7+1m8zkH1yruvplEjY=", + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/google-logging-utils": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", @@ -4188,6 +4819,24 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha1-4193ADU1sCl+oIVI9azmrbFIDdw=", + "license": "ISC" + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha1-lj7X0HHce/XwhMW/vg0bYiJYaFQ=", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -4458,6 +5107,12 @@ "integrity": "sha1-6Y7nsYmf9KGEU00fFnwojGa77/Q=", "license": "BSD-2-Clause" }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "license": "ISC" + }, "node_modules/jsonwebtoken": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", @@ -4604,6 +5259,18 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha1-vZBg9MW3CqgEHMxvgDaHYJlPMMo=", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -4687,6 +5354,18 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha1-atdsOo8QInybUdHJrI4wsn9aJRw=", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/module-details-from-path": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", @@ -4806,6 +5485,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha1-HEfyct8nfzsdrwYWd9nILiMixg4=", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/obug": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", @@ -4850,6 +5538,49 @@ "fn.name": "1.x.x" } }, + "node_modules/onnxruntime-common": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz", + "integrity": "sha1-qB1BkdQYrLv/JUapVMwswj7rCfg=", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz", + "integrity": "sha1-f09ZRVuvhRGB4gf8hAEoisLrENE=", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "global-agent": "^3.0.0", + "onnxruntime-common": "1.21.0", + "tar": "^7.0.1" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha1-0eOgTgPf7jkrQdQg71R7agNRsGs=", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha1-PUo5VjuT2z0EKLVSfLpYo8j4JsI=", + "license": "MIT" + }, "node_modules/open": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", @@ -4986,6 +5717,12 @@ "node": ">=16.20.0" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha1-SLTOmDFksgnC1FoQetsx9HOm56c=", + "license": "MIT" + }, "node_modules/postcss": { "version": "8.5.28", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", @@ -5194,6 +5931,23 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha1-9f55W3uDjM/jXcYI4Cgrnrouev0=", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/rollup": { "version": "4.63.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.1.tgz", @@ -5308,7 +6062,6 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha1-ObZGA33VDBT7RR5+TKxY7YuGP2k=", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -5317,6 +6070,12 @@ "node": ">=10" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", + "license": "MIT" + }, "node_modules/send": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", @@ -5343,6 +6102,21 @@ "url": "https://opencollective.com/express" } }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha1-8TYLBEf2H/tIPsQVfHN/q313jhg=", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serve-static": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", @@ -5368,6 +6142,50 @@ "integrity": "sha1-ZsmiSnP5/CjL5msJ/tPTPcrxtCQ=", "license": "ISC" }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha1-tvFI5LjGHxeXveEanRz+u64sV7A=", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -5490,6 +6308,12 @@ "node": ">=0.10.0" } }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha1-SRS5A6L4toXRf994pw6RfocuREo=", + "license": "BSD-3-Clause" + }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -5653,6 +6477,22 @@ "url": "https://www.buymeacoffee.com/systeminfo" } }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha1-ppb5mBNucUh9w/hpqFu6LGeXG6k=", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -5725,9 +6565,21 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha1-YS7+TtI11Wfoq6Xypfq3AoCt6D8=", - "dev": true, + "devOptional": true, "license": "0BSD" }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha1-AXLLW86AsL1ULqNI21DH4hg02TQ=", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/type-is": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", @@ -6204,6 +7056,15 @@ "node": ">=10" } }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha1-AOLeRDY57Q14/YfeDSdGn7z/tTM=", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", diff --git a/package.json b/package.json index 12ff747..3ee2ce8 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,9 @@ "test:integration": "vitest run test/integration", "test:watch": "vitest --watch", "test:ui": "vitest --ui", - "clean": "rm -rf dist" + "clean": "rm -rf dist", + "semantic:build": "node dist/semantic/build-index.js", + "semantic:query": "node dist/semantic/query.js" }, "keywords": [ "mcp", @@ -31,6 +33,7 @@ "author": "Microsoft", "license": "MIT", "dependencies": { + "@huggingface/transformers": "^3.8.1", "@modelcontextprotocol/sdk": "^1.30.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/api-logs": "^0.222.0", diff --git a/src/semantic/README.md b/src/semantic/README.md new file mode 100644 index 0000000..4394450 --- /dev/null +++ b/src/semantic/README.md @@ -0,0 +1,70 @@ +# Semantic search prototype (local-only, advisory-specific) + +> Prototype on branch `proto/semantic-search`. Not wired into the MCP tools yet. +> No external engine or service; runs entirely on-device. + +Hybrid retrieval for advisories: local embeddings + BM25, fused with Reciprocal +Rank Fusion, then a field-aware rerank. Advisory-specific (uses GHSA/CVE ids, +summary, CWEs, affected packages, ecosystem, severity) — not a general-purpose +memory store. + +## Pieces + +| File | Role | +|---|---| +| `document.ts` | Advisory → searchable doc (embedded text + rerank fields) | +| `embeddings.ts` | Local ONNX embeddings (`@huggingface/transformers`, MiniLM 384-dim), offline | +| `bm25.ts` | Compact Okapi BM25, identifier-preserving tokenizer | +| `temporal.ts` | Parse a period from the query + score publish-date proximity | +| `store.ts` | Persist/load `embeddings.bin` + `bm25.json` + `docs.json` + `meta.json` | +| `hybrid.ts` | Vector kNN ∪ BM25 → RRF → field-aware + temporal rerank | +| `build-index.ts` | CLI: build the index from the advisory DB | +| `query.ts` | CLI: query the index | + +## Run + +```powershell +# reuse an existing transformers cache to stay fully offline (no HF download) +$env:SEMANTIC_MODEL_CACHE = "C:\build\romulus-gym\services\memory\models\.cache" +$env:ADVISORY_REPO_PATH = "./external/advisory-database" + +npm run build +node dist/semantic/build-index.js --limit 5000 # omit --limit for all reviewed +node dist/semantic/query.js "blind ORM injection via sort parameter" --top 8 +``` + +## Config (env) + +- `SEMANTIC_MODEL` (default `Xenova/all-MiniLM-L6-v2`), `SEMANTIC_DIM` (384) +- `SEMANTIC_MODEL_CACHE` — transformers cache dir (point at a pre-downloaded cache) +- `SEMANTIC_ALLOW_REMOTE=true` — allow HF download (default offline) +- `SEMANTIC_INDEX_DIR` (default `./.semantic-index`), `SEMANTIC_BATCH` (64) + +## Temporal-aware reranking + +If a query mentions a period, that intent is parsed out (`temporal.ts`), the +*residual* text drives semantic + BM25 recall, and a proximity score reranks by +how close each advisory's `published` date is to the window: + +- Recognized: explicit range `2026-01-01..2026-06-30`, ISO date, `2026-08`, + month names (`August 2026`), `last/past N days|weeks|months|years`, + `this/last week|month|year`, `today`/`yesterday`, bare year, and fuzzy + `recent|latest` (→ last 90 days). +- In-window → 1.0; outside → exponential decay (half-life 45 days), added with a + small weight so it reorders *within* relevant results rather than becoming a + hard date sort. Example: `"sql injection in june 2026"` lifts June/early-July + advisories above later ones while every hit stays SQL-injection. +- A hard date filter still lives in the structured `published`/`updated` params. + +## Design notes / next steps + +- **Storage** is a flat `Float32Array` + brute-force cosine — fine for the ~35k + reviewed tier (<50 ms). For the 370k unreviewed tier, swap in ANN (hnswlib). +- **Reranking** is currently field-aware (exact id / package / CWE / phrase). The + `rerankScore` interface can be replaced by a local cross-encoder (e.g. + `Xenova/bge-reranker-base`) with no other changes. +- **Incremental**: rebuild only changed advisories via `git diff` between the + cached `dbCommit` and current `HEAD` (not yet implemented). +- **Not wired to MCP**: a `semantic_search` tool that combines this with the + structured filters (`ecosystem`, `cwes`, `web_app_only`, date ranges) is the + intended next step. diff --git a/src/semantic/bm25.ts b/src/semantic/bm25.ts new file mode 100644 index 0000000..8fc5039 --- /dev/null +++ b/src/semantic/bm25.ts @@ -0,0 +1,94 @@ +/** + * Compact in-process BM25 (Okapi) — no external engine. + * + * Advisory-tuned: identifiers (GHSA-…, CVE-…, package names) survive tokenization + * so exact-id and package queries score strongly. + */ + +const K1 = 1.5; +const B = 0.75; + +export function tokenize(text: string): string[] { + return text + .toLowerCase() + // keep letters/digits/hyphen/dot/underscore so "ghsa-xxxx", "cve-2026-1", "next.js" stay intact + .split(/[^a-z0-9._-]+/) + .filter(t => t.length > 1); +} + +export interface Bm25Data { + ids: string[]; + df: Record; + docTokens: number[][]; // per-doc term-id lists (indexes into `vocab`) + vocab: string[]; // term-id -> term + tf: Record[]; // per-doc term-id -> frequency + avgdl: number; + n: number; +} + +export class Bm25 { + private data: Bm25Data; + private termId: Map; + + constructor(data: Bm25Data) { + this.data = data; + this.termId = new Map(data.vocab.map((t, i) => [t, i])); + } + + static build(ids: string[], texts: string[]): Bm25 { + const vocab: string[] = []; + const termId = new Map(); + const df: Record = {}; + const tf: Record[] = []; + const docTokens: number[][] = []; + let total = 0; + + for (const text of texts) { + const toks = tokenize(text); + total += toks.length; + const counts: Record = {}; + const seen = new Set(); + const ids2: number[] = []; + for (const tok of toks) { + let id = termId.get(tok); + if (id === undefined) { id = vocab.length; vocab.push(tok); termId.set(tok, id); } + counts[id] = (counts[id] || 0) + 1; + ids2.push(id); + if (!seen.has(id)) { seen.add(id); df[tok] = (df[tok] || 0) + 1; } + } + tf.push(counts); + docTokens.push(ids2); + } + + const n = texts.length; + const data: Bm25Data = { ids, df, docTokens, vocab, tf, avgdl: total / (n || 1), n }; + return new Bm25(data); + } + + toJSON(): Bm25Data { return this.data; } + static fromJSON(d: Bm25Data): Bm25 { return new Bm25(d); } + + /** Returns [docIndex, score] pairs sorted desc, top `limit`. */ + search(query: string, limit: number): Array<[number, number]> { + const { df, tf, docTokens, avgdl, n } = this.data; + const qTerms = tokenize(query); + const scores = new Map(); + + for (const term of qTerms) { + const id = this.termId.get(term); + if (id === undefined) continue; + const termDf = df[term] || 0; + if (termDf === 0) continue; + const idf = Math.log(1 + (n - termDf + 0.5) / (termDf + 0.5)); + for (let d = 0; d < n; d++) { + const f = tf[d][id]; + if (!f) continue; + const dl = docTokens[d].length; + const denom = f + K1 * (1 - B + B * (dl / avgdl)); + scores.set(d, (scores.get(d) || 0) + idf * ((f * (K1 + 1)) / denom)); + } + } + + return [...scores.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit); + } +} diff --git a/src/semantic/build-index.ts b/src/semantic/build-index.ts new file mode 100644 index 0000000..f1c6ce3 --- /dev/null +++ b/src/semantic/build-index.ts @@ -0,0 +1,57 @@ +/** + * Prototype CLI: build the local hybrid index from the advisory DB. + * + * ADVISORY_REPO_PATH=./external/advisory-database \ + * SEMANTIC_MODEL_CACHE=C:/build/romulus-gym/services/memory/models/.cache \ + * node dist/semantic/build-index.js --limit 5000 + * + * --limit N build over the first N advisories (fast validation); omit for all reviewed. + */ + +import { execFileSync } from 'child_process'; +import { join } from 'path'; +import { LocalRepositoryDataSource } from '../datasources/local-repository.js'; +import { toDoc, type AdvisoryDoc } from './document.js'; +import { embedBatch, modelName, EMBED_DIM } from './embeddings.js'; +import { Bm25 } from './bm25.js'; +import { saveIndex, INDEX_DIR, type IndexMeta } from './store.js'; + +const REPO = process.env.ADVISORY_REPO_PATH || './external/advisory-database'; +const BATCH = Number(process.env.SEMANTIC_BATCH || 64); + +function dbCommit(): string { + try { + return execFileSync('git', ['-C', REPO, 'rev-parse', '--short', 'HEAD'], { encoding: 'utf-8' }).trim(); + } catch { return 'unknown'; } +} + +async function main() { + const limitArg = process.argv.indexOf('--limit'); + const limit = limitArg !== -1 ? Number(process.argv[limitArg + 1]) : Infinity; + + console.error(`[build] repo=${REPO} model=${modelName()} dim=${EMBED_DIM} limit=${limit}`); + const ds = new LocalRepositoryDataSource(REPO); + const advisories = await ds.listAdvisories({ type: 'reviewed', per_page: Number.isFinite(limit) ? limit : 1_000_000 }); + const docs: AdvisoryDoc[] = advisories.map(toDoc); + console.error(`[build] documents: ${docs.length}`); + + const vectors: Float32Array[] = []; + const t0 = Date.now(); + for (let i = 0; i < docs.length; i += BATCH) { + const batch = docs.slice(i, i + BATCH).map(d => d.text); + vectors.push(...await embedBatch(batch)); + if (i % (BATCH * 20) === 0) console.error(`[build] embedded ${i + batch.length}/${docs.length}`); + } + console.error(`[build] embeddings done in ${((Date.now() - t0) / 1000).toFixed(1)}s`); + + const bm25 = Bm25.build(docs.map(d => d.id), docs.map(d => d.text)); + + const meta: IndexMeta = { + model: modelName(), dim: EMBED_DIM, count: docs.length, + dbCommit: dbCommit(), builtAt: new Date().toISOString(), + }; + await saveIndex(meta, docs, vectors, bm25.toJSON()); + console.error(`[build] saved index -> ${join(INDEX_DIR)} (${docs.length} docs)`); +} + +main().catch(err => { console.error(err); process.exit(1); }); diff --git a/src/semantic/document.ts b/src/semantic/document.ts new file mode 100644 index 0000000..93ca74c --- /dev/null +++ b/src/semantic/document.ts @@ -0,0 +1,59 @@ +/** + * Advisory -> searchable document. Prototype, advisory-specific (no generic memory). + * + * We keep two views of each advisory: + * - `text` : the string we embed and BM25-tokenize (semantic + lexical recall) + * - `fields` : structured fields the reranker uses for exact/near-exact boosts + */ + +import type { Advisory } from '../types/data-source.js'; + +export interface AdvisoryDoc { + id: string; // ghsa_id + text: string; // embedded + tokenized + ghsa_id: string; + cve_id: string; + summary: string; + cweIds: string[]; // e.g. ["CWE-89"] + packages: string[]; // affected package names (lowercased) + ecosystems: string[]; // OSV ecosystem names (lowercased) + severity: string; + published: number; // epoch ms (0 if unknown) — used for temporal rerank + updated: number; // epoch ms (0 if unknown) +} + +const DETAILS_MAX = 500; // embedding models truncate ~512 tokens; keep the tail cheap + +export function toDoc(a: Advisory): AdvisoryDoc { + const cweIds = (a.cwes || []).map(c => c.cwe_id); + const packages = (a.vulnerabilities || []).map(v => v.package?.name).filter(Boolean) as string[]; + const ecosystems = (a.vulnerabilities || []).map(v => v.package?.ecosystem).filter(Boolean) as string[]; + const cweNames = cweIds.join(' '); + const details = (a.description || '').slice(0, DETAILS_MAX); + + // Field order matters for lexical weighting: summary first, then identifiers, + // then the taxonomy/package signals, then a slice of the details. + const text = [ + a.summary || '', + a.ghsa_id, + a.cve_id || '', + packages.join(' '), + ecosystems.join(' '), + cweNames, + details, + ].filter(Boolean).join('\n'); + + return { + id: a.ghsa_id, + text, + ghsa_id: a.ghsa_id, + cve_id: a.cve_id || '', + summary: a.summary || '', + cweIds, + packages: packages.map(p => p.toLowerCase()), + ecosystems: ecosystems.map(e => e.toLowerCase()), + severity: (a.severity || '').toLowerCase(), + published: a.published_at ? Date.parse(a.published_at) || 0 : 0, + updated: a.updated_at ? Date.parse(a.updated_at) || 0 : 0, + }; +} diff --git a/src/semantic/embeddings.ts b/src/semantic/embeddings.ts new file mode 100644 index 0000000..397d17e --- /dev/null +++ b/src/semantic/embeddings.ts @@ -0,0 +1,57 @@ +/** + * Local, offline embeddings via @huggingface/transformers (ONNX, CPU). + * + * No network at query time and no external service. Defaults to the cached + * all-MiniLM-L6-v2 (384-dim). Point SEMANTIC_MODEL_CACHE at an existing + * transformers cache to avoid any download on locked-down devices. + */ + +import { env, pipeline } from '@huggingface/transformers'; + +const MODEL = process.env.SEMANTIC_MODEL || 'Xenova/all-MiniLM-L6-v2'; +export const EMBED_DIM = Number(process.env.SEMANTIC_DIM || 384); + +// Offline by default; reuse an existing cache dir if provided. +env.allowRemoteModels = process.env.SEMANTIC_ALLOW_REMOTE === 'true'; +if (process.env.SEMANTIC_MODEL_CACHE) { + env.cacheDir = process.env.SEMANTIC_MODEL_CACHE; +} + +let extractor: any | null = null; + +async function getExtractor(): Promise { + if (!extractor) { + extractor = await pipeline('feature-extraction', MODEL); + } + return extractor; +} + +export function modelName(): string { + return MODEL; +} + +/** Embed a single text -> unit-normalized Float32Array[EMBED_DIM]. */ +export async function embed(text: string): Promise { + const ex = await getExtractor(); + const out = await ex(text, { pooling: 'mean', normalize: true }); + return new Float32Array(out.data); +} + +/** Batch embed -> one ONNX forward pass; returns rows of EMBED_DIM. */ +export async function embedBatch(texts: string[]): Promise { + if (texts.length === 0) return []; + if (texts.length === 1) return [await embed(texts[0])]; + const ex = await getExtractor(); + const out = await ex(texts, { pooling: 'mean', normalize: true }); + const data = out.data as Float32Array; + return Array.from({ length: texts.length }, (_, i) => + data.slice(i * EMBED_DIM, (i + 1) * EMBED_DIM) + ); +} + +/** Cosine of two unit vectors == dot product. */ +export function cosine(a: Float32Array, b: Float32Array): number { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i] * b[i]; + return s; +} diff --git a/src/semantic/hybrid.ts b/src/semantic/hybrid.ts new file mode 100644 index 0000000..253aba8 --- /dev/null +++ b/src/semantic/hybrid.ts @@ -0,0 +1,111 @@ +/** + * Hybrid retrieval: BM25 + dense vectors fused with Reciprocal Rank Fusion, + * then a lightweight, field-aware rerank. All local; no cross-encoder needed + * (a local reranker model can replace `rerankScore` later behind this interface). + */ + +import { embed, cosine, EMBED_DIM } from './embeddings.js'; +import { Bm25, tokenize } from './bm25.js'; +import { parseTemporal, temporalRelevance } from './temporal.js'; +import type { LoadedIndex, StoredDoc } from './store.js'; + +export interface SearchHit { + ghsa_id: string; + cve_id: string; + summary: string; + severity: string; + cweIds: string[]; + packages: string[]; + published: number; + scores: { rrf: number; vector: number; bm25Rank: number; rerank: number; temporal: number; final: number }; +} + +const RRF_K = 60; +const TEMPORAL_WEIGHT = 0.02; // additive; comparable to an RRF top-rank contribution + +function vectorTopK(query: Float32Array, vectors: Float32Array, n: number, k: number): Array<[number, number]> { + const scored: Array<[number, number]> = []; + for (let i = 0; i < n; i++) { + const row = vectors.subarray(i * EMBED_DIM, (i + 1) * EMBED_DIM); + scored.push([i, cosine(query, row)]); + } + scored.sort((a, b) => b[1] - a[1]); + return scored.slice(0, k); +} + +/** + * Field-aware rerank of a fused candidate. Rewards exact identifiers, affected + * package names present in the query, CWE mentions, and summary phrase overlap. + */ +function rerankScore(query: string, qTokens: Set, doc: StoredDoc): number { + const q = query.toLowerCase(); + let s = 0; + if (doc.ghsa_id && q.includes(doc.ghsa_id.toLowerCase())) s += 5; + if (doc.cve_id && q.includes(doc.cve_id.toLowerCase())) s += 5; + if (doc.packages.some(p => qTokens.has(p))) s += 2; + if (doc.cweIds.some(c => qTokens.has(c.toLowerCase()) || qTokens.has(c.replace(/^cwe-/i, '')))) s += 1.5; + const summaryTokens = new Set(tokenize(doc.summary)); + let overlap = 0; + for (const t of qTokens) if (summaryTokens.has(t)) overlap++; + s += Math.min(overlap, 5) * 0.4; + return s; +} + +export async function hybridSearch( + index: LoadedIndex, + query: string, + topK = 10, + fetch = 50 +): Promise { + const bm25 = Bm25.fromJSON(index.bm25); + + // Split temporal intent out of the query: recall runs on the residual text, + // the period drives a proximity boost in the rerank stage. + const temporal = parseTemporal(query); + const recallQuery = temporal.present && temporal.residual ? temporal.residual : query; + + const qEmbedding = await embed(recallQuery); + const qTokens = new Set(tokenize(recallQuery)); + + const vec = vectorTopK(qEmbedding, index.vectors, index.docs.length, fetch); + const lex = bm25.search(recallQuery, fetch); + + // Reciprocal Rank Fusion over the two candidate lists. + const rrf = new Map(); + const vecScore = new Map(); + vec.forEach(([d, s], rank) => { rrf.set(d, (rrf.get(d) || 0) + 1 / (RRF_K + rank + 1)); vecScore.set(d, s); }); + const bmRank = new Map(); + lex.forEach(([d], rank) => { rrf.set(d, (rrf.get(d) || 0) + 1 / (RRF_K + rank + 1)); bmRank.set(d, rank + 1); }); + + // Rerank the fused candidates. + const candidates = [...rrf.keys()]; + const hits = candidates.map(d => { + const doc = index.docs[d]; + const rerank = rerankScore(recallQuery, qTokens, doc); + const rrfScore = rrf.get(d)!; + const temporalScore = temporalRelevance(doc.published, temporal); + return { + d, + hit: { + ghsa_id: doc.ghsa_id, + cve_id: doc.cve_id, + summary: doc.summary, + severity: doc.severity, + cweIds: doc.cweIds, + packages: doc.packages, + published: doc.published, + scores: { + rrf: rrfScore, + vector: vecScore.get(d) ?? 0, + bm25Rank: bmRank.get(d) ?? 0, + rerank, + temporal: temporalScore, + final: rrfScore + rerank * 0.01 + temporalScore * TEMPORAL_WEIGHT, + }, + } as SearchHit, + }; + }); + + hits.sort((a, b) => b.hit.scores.final - a.hit.scores.final); + return hits.slice(0, topK).map(h => h.hit); +} diff --git a/src/semantic/query.ts b/src/semantic/query.ts new file mode 100644 index 0000000..b6f5bb0 --- /dev/null +++ b/src/semantic/query.ts @@ -0,0 +1,34 @@ +/** + * Prototype CLI: query the local hybrid index. + * + * SEMANTIC_MODEL_CACHE=C:/build/romulus-gym/services/memory/models/.cache \ + * node dist/semantic/query.js "blind ORM injection via sort parameter" --top 8 + */ + +import { indexExists, loadIndex } from './store.js'; +import { hybridSearch } from './hybrid.js'; + +async function main() { + const args = process.argv.slice(2); + const topIdx = args.indexOf('--top'); + const top = topIdx !== -1 ? Number(args[topIdx + 1]) : 10; + const query = args.filter((a, i) => a !== '--top' && args[i - 1] !== '--top').join(' ').trim(); + + if (!query) { console.error('usage: query.js "" [--top N]'); process.exit(2); } + if (!indexExists()) { console.error('No index found. Run build-index.js first.'); process.exit(1); } + + const index = await loadIndex(); + console.error(`[query] index: ${index.meta.count} docs, model ${index.meta.model}, DB ${index.meta.dbCommit}`); + const t0 = Date.now(); + const hits = await hybridSearch(index, query, top); + console.error(`[query] "${query}" -> ${hits.length} hits in ${Date.now() - t0}ms\n`); + + hits.forEach((h, i) => { + const pub = h.published ? new Date(h.published).toISOString().slice(0, 10) : '----------'; + console.log(`${(i + 1).toString().padStart(2)}. ${h.ghsa_id} [${h.severity}] pub=${pub} ${h.summary}`); + console.log(` cwes=${h.cweIds.join(',') || '-'} pkgs=${h.packages.slice(0, 4).join(',') || '-'}`); + console.log(` final=${h.scores.final.toFixed(4)} rrf=${h.scores.rrf.toFixed(4)} vec=${h.scores.vector.toFixed(3)} bm25Rank=${h.scores.bm25Rank} rerank=${h.scores.rerank.toFixed(1)} temporal=${h.scores.temporal.toFixed(2)}`); + }); +} + +main().catch(err => { console.error(err); process.exit(1); }); diff --git a/src/semantic/store.ts b/src/semantic/store.ts new file mode 100644 index 0000000..81135ac --- /dev/null +++ b/src/semantic/store.ts @@ -0,0 +1,61 @@ +/** + * On-disk index: embeddings.bin (Float32) + bm25.json + docs.json + meta.json. + * Git-ignored; rebuilt from the advisory DB. No external store. + */ + +import { readFile, writeFile, mkdir } from 'fs/promises'; +import { existsSync } from 'fs'; +import { join } from 'path'; +import { EMBED_DIM } from './embeddings.js'; +import type { AdvisoryDoc } from './document.js'; +import type { Bm25Data } from './bm25.js'; + +export const INDEX_DIR = process.env.SEMANTIC_INDEX_DIR || './.semantic-index'; + +export interface IndexMeta { + model: string; + dim: number; + count: number; + dbCommit: string; + builtAt: string; +} + +// Fields kept for rerank/display (drop the heavy `text`). +export type StoredDoc = Omit; + +export interface LoadedIndex { + meta: IndexMeta; + docs: StoredDoc[]; + vectors: Float32Array; // count * dim, row-major + bm25: Bm25Data; +} + +export async function saveIndex( + meta: IndexMeta, + docs: AdvisoryDoc[], + vectors: Float32Array[], + bm25: Bm25Data +): Promise { + await mkdir(INDEX_DIR, { recursive: true }); + const flat = new Float32Array(vectors.length * EMBED_DIM); + vectors.forEach((v, i) => flat.set(v, i * EMBED_DIM)); + await writeFile(join(INDEX_DIR, 'embeddings.bin'), Buffer.from(flat.buffer)); + const stored: StoredDoc[] = docs.map(({ text, ...rest }) => rest); + await writeFile(join(INDEX_DIR, 'docs.json'), JSON.stringify(stored)); + await writeFile(join(INDEX_DIR, 'bm25.json'), JSON.stringify(bm25)); + await writeFile(join(INDEX_DIR, 'meta.json'), JSON.stringify(meta, null, 2)); +} + +export function indexExists(): boolean { + return ['embeddings.bin', 'docs.json', 'bm25.json', 'meta.json'] + .every(f => existsSync(join(INDEX_DIR, f))); +} + +export async function loadIndex(): Promise { + const meta: IndexMeta = JSON.parse(await readFile(join(INDEX_DIR, 'meta.json'), 'utf-8')); + const docs: StoredDoc[] = JSON.parse(await readFile(join(INDEX_DIR, 'docs.json'), 'utf-8')); + const bm25: Bm25Data = JSON.parse(await readFile(join(INDEX_DIR, 'bm25.json'), 'utf-8')); + const buf = await readFile(join(INDEX_DIR, 'embeddings.bin')); + const vectors = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4); + return { meta, docs, vectors, bm25 }; +} diff --git a/src/semantic/temporal.ts b/src/semantic/temporal.ts new file mode 100644 index 0000000..e8d46cb --- /dev/null +++ b/src/semantic/temporal.ts @@ -0,0 +1,101 @@ +/** + * Parse temporal intent from a free-text query and score how close an advisory's + * publish date is to that period. Soft signal for reranking — not a hard filter. + */ + +export interface Temporal { + present: boolean; // did the query express a period? + start: number; // epoch ms, inclusive + end: number; // epoch ms, exclusive + residual: string; // query with the temporal phrase removed (for embed/BM25) +} + +const DAY = 86_400_000; +const MONTHS: Record = { + jan: 0, january: 0, feb: 1, february: 1, mar: 2, march: 2, apr: 3, april: 3, + may: 4, jun: 5, june: 5, jul: 6, july: 6, aug: 7, august: 7, sep: 8, sept: 8, + september: 8, oct: 9, october: 9, nov: 10, november: 10, dec: 11, december: 11, +}; + +const utc = (y: number, m = 0, d = 1) => Date.UTC(y, m, d); + +/** Try each pattern; on match, return the window + the query minus the phrase. */ +export function parseTemporal(query: string, now: number = Date.now()): Temporal { + const none: Temporal = { present: false, start: 0, end: 0, residual: query }; + const strip = (re: RegExp): string => query.replace(re, ' ').replace(/\s+/g, ' ').trim(); + let m: RegExpMatchArray | null; + + // 1) explicit range YYYY-MM-DD..YYYY-MM-DD + if ((m = query.match(/\b(\d{4}-\d{2}-\d{2})\s*\.\.\s*(\d{4}-\d{2}-\d{2})\b/))) { + return { present: true, start: Date.parse(m[1] + 'T00:00:00Z'), end: Date.parse(m[2] + 'T00:00:00Z') + DAY, residual: strip(/\b\d{4}-\d{2}-\d{2}\s*\.\.\s*\d{4}-\d{2}-\d{2}\b/) }; + } + // 2) single ISO date + if ((m = query.match(/\b(\d{4})-(\d{2})-(\d{2})\b/))) { + const s = Date.parse(`${m[1]}-${m[2]}-${m[3]}T00:00:00Z`); + return { present: true, start: s, end: s + DAY, residual: strip(/\b\d{4}-\d{2}-\d{2}\b/) }; + } + // 3) YYYY-MM + if ((m = query.match(/\b(\d{4})-(\d{2})\b/))) { + const y = +m[1], mo = +m[2] - 1; + return { present: true, start: utc(y, mo), end: utc(y, mo + 1), residual: strip(/\b\d{4}-\d{2}\b/) }; + } + // 4) month name + year ("August 2026", "aug 2026") + if ((m = query.match(/\b([A-Za-z]{3,9})\.?\s+(\d{4})\b/)) && MONTHS[m[1].toLowerCase()] !== undefined) { + const mo = MONTHS[m[1].toLowerCase()], y = +m[2]; + return { present: true, start: utc(y, mo), end: utc(y, mo + 1), residual: strip(/\b[A-Za-z]{3,9}\.?\s+\d{4}\b/) }; + } + // 5) "last/past N days|weeks|months|years" + if ((m = query.match(/\b(?:last|past|within)\s+(\d+)\s+(day|week|month|year)s?\b/i))) { + const n = +m[1], unit = m[2].toLowerCase(); + const span = unit === 'day' ? n * DAY : unit === 'week' ? n * 7 * DAY : unit === 'month' ? n * 30 * DAY : n * 365 * DAY; + return { present: true, start: now - span, end: now + DAY, residual: strip(/\b(?:last|past|within)\s+\d+\s+(?:day|week|month|year)s?\b/i) }; + } + // 6) "this|last week|month|year" + if ((m = query.match(/\b(this|last)\s+(week|month|year)\b/i))) { + const which = m[1].toLowerCase(), unit = m[2].toLowerCase(); + const d = new Date(now); + let start: number, end: number; + if (unit === 'year') { + const y = d.getUTCFullYear() - (which === 'last' ? 1 : 0); + start = utc(y, 0); end = which === 'last' ? utc(y + 1, 0) : now + DAY; + } else if (unit === 'month') { + const y = d.getUTCFullYear(), mo = d.getUTCMonth() - (which === 'last' ? 1 : 0); + start = utc(y, mo); end = which === 'last' ? utc(y, mo + 1) : now + DAY; + } else { // week + const span = 7 * DAY; + start = which === 'last' ? now - 2 * span : now - span; end = which === 'last' ? now - span : now + DAY; + } + return { present: true, start, end, residual: strip(/\b(?:this|last)\s+(?:week|month|year)\b/i) }; + } + // 7) today / yesterday + if (/\byesterday\b/i.test(query)) { + const midnight = Math.floor(now / DAY) * DAY; + return { present: true, start: midnight - DAY, end: midnight, residual: strip(/\byesterday\b/i) }; + } + if (/\btoday\b/i.test(query)) { + const midnight = Math.floor(now / DAY) * DAY; + return { present: true, start: midnight, end: now + DAY, residual: strip(/\btoday\b/i) }; + } + // 8) bare year (e.g. "2026") + if ((m = query.match(/\b(19|20)\d{2}\b/))) { + const y = +m[0]; + return { present: true, start: utc(y, 0), end: utc(y + 1, 0), residual: strip(/\b(?:19|20)\d{2}\b/) }; + } + // 9) fuzzy "recent|recently|latest|newest" -> last 90 days (decay from now) + if (/\b(recent|recently|latest|newest|new)\b/i.test(query)) { + return { present: true, start: now - 90 * DAY, end: now + DAY, residual: strip(/\b(?:recent|recently|latest|newest|new)\b/i) }; + } + + return none; +} + +/** + * 1.0 inside the window; exponential decay by distance (days) to the nearest + * boundary outside it. Unknown dates score 0. + */ +export function temporalRelevance(publishedMs: number, t: Temporal, halfLifeDays = 45): number { + if (!t.present || !publishedMs) return 0; + if (publishedMs >= t.start && publishedMs < t.end) return 1; + const distDays = (publishedMs < t.start ? t.start - publishedMs : publishedMs - t.end) / DAY; + return Math.exp(-distDays / halfLifeDays); +} From 6013b0e1f1e0297a44f844b76d49ff3c718a0113 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 20:15:31 -0700 Subject: [PATCH 08/19] proto(semantic): expose semantic_search MCP tool Registers semantic_search in createAdvisoryServer (stdio + HTTP): hybrid local search with optional web_app_only/severity/ecosystem/cwes post-filters and temporal reranking. Returns build instructions if the index is absent. --- src/semantic/hybrid.ts | 2 + src/server.ts | 19 ++++++++ src/tools/semantic-search.ts | 89 ++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 src/tools/semantic-search.ts diff --git a/src/semantic/hybrid.ts b/src/semantic/hybrid.ts index 253aba8..e1fc4e4 100644 --- a/src/semantic/hybrid.ts +++ b/src/semantic/hybrid.ts @@ -16,6 +16,7 @@ export interface SearchHit { severity: string; cweIds: string[]; packages: string[]; + ecosystems: string[]; published: number; scores: { rrf: number; vector: number; bm25Rank: number; rerank: number; temporal: number; final: number }; } @@ -93,6 +94,7 @@ export async function hybridSearch( severity: doc.severity, cweIds: doc.cweIds, packages: doc.packages, + ecosystems: doc.ecosystems, published: doc.published, scores: { rrf: rrfScore, diff --git a/src/server.ts b/src/server.ts index 94cf2e8..3d44047 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4,6 +4,7 @@ import { listAdvisories, getAdvisory, } from "./tools/advisories.js"; +import { semanticSearch } from "./tools/semantic-search.js"; /** * Create and configure the MCP server for local advisory database @@ -61,6 +62,22 @@ export function createAdvisoryServer(): Server { }, required: ["ghsa_id"] } + }, + { + name: "semantic_search", + description: "Local hybrid semantic search over advisories (embeddings + BM25 + reranking). Use for natural-language / conceptual queries (e.g. 'blind ORM injection via sort parameter', 'account takeover in recent Keycloak'). A period in the query ('in August 2026', 'recent') drives temporal reranking. Requires the prototype index to be built.", + inputSchema: { + type: "object", + properties: { + query: { type: "string", description: "Natural-language query; may include a period for temporal reranking" }, + top_k: { type: "number", minimum: 1, maximum: 50, description: "Number of results (default 10)" }, + web_app_only: { type: "boolean", description: "Keep only web-application vulnerability classes (by CWE)" }, + severity: { type: "string", enum: ['low', 'medium', 'high', 'critical', 'unknown'], description: "Post-filter by severity" }, + ecosystem: { type: "string", enum: ['rubygems', 'npm', 'pip', 'maven', 'nuget', 'composer', 'go', 'rust', 'erlang', 'actions', 'pub', 'other', 'swift'], description: "Post-filter by ecosystem" }, + cwes: { type: "string", description: "Comma-separated CWE ids to require (e.g. '89' or 'CWE-89,79')" } + }, + required: ["query"] + } } ] }; @@ -75,6 +92,8 @@ export function createAdvisoryServer(): Server { return await listAdvisories(args || {}); case "get_advisory": return await getAdvisory(args || {}); + case "semantic_search": + return await semanticSearch(args || {}); default: throw new Error(`Unknown tool: ${name}`); } diff --git a/src/tools/semantic-search.ts b/src/tools/semantic-search.ts new file mode 100644 index 0000000..71a1533 --- /dev/null +++ b/src/tools/semantic-search.ts @@ -0,0 +1,89 @@ +/** + * semantic_search tool — local hybrid retrieval over the prototype index. + * Loads the on-disk index + embedding model in-process; no external service. + */ + +import { z } from 'zod'; +import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; +import { createLogger } from '../logger.js'; +import { indexExists, loadIndex, INDEX_DIR, type LoadedIndex } from '../semantic/store.js'; +import { hybridSearch } from '../semantic/hybrid.js'; +import { ecosystemMatches, cweFilterMatches, isWebAppAdvisory } from '../datasources/local-repository.js'; + +const logger = createLogger('Tools'); + +export const semanticSearchSchema = z.object({ + query: z.string().describe('Natural-language query. May include a period ("SSRF in August 2026", "recent RCE") — the date drives temporal reranking.'), + top_k: z.number().min(1).max(50).optional().describe('Number of results (default 10)'), + web_app_only: z.boolean().optional().describe('Keep only web-application vulnerability classes (by CWE)'), + severity: z.enum(['low', 'medium', 'high', 'critical', 'unknown']).optional().describe('Post-filter by severity'), + ecosystem: z.enum(['rubygems', 'npm', 'pip', 'maven', 'nuget', 'composer', 'go', 'rust', 'erlang', 'actions', 'pub', 'other', 'swift']).optional().describe('Post-filter by package ecosystem'), + cwes: z.string().optional().describe('Comma-separated CWE ids to require (e.g. "89" or "CWE-89,79")'), +}); + +let cachedIndex: LoadedIndex | null = null; + +async function getIndex(): Promise { + if (cachedIndex) return cachedIndex; + if (!indexExists()) return null; + cachedIndex = await loadIndex(); + return cachedIndex; +} + +export async function semanticSearch(params: unknown): Promise { + const p = semanticSearchSchema.parse(params ?? {}); + logger.debug('semantic_search called', { query: p.query }); + + const index = await getIndex(); + if (!index) { + return { + isError: true, + content: [{ + type: 'text', + text: `Semantic index not found at ${INDEX_DIR}. Build it first:\n` + + ` ADVISORY_REPO_PATH=./external/advisory-database \\\n` + + ` SEMANTIC_MODEL_CACHE= \\\n` + + ` node dist/semantic/build-index.js --limit 5000`, + }], + }; + } + + const topK = p.top_k ?? 10; + const cweReq = p.cwes ? [p.cwes] : null; + + // Over-fetch, then apply structured post-filters, then slice. + const raw = await hybridSearch(index, p.query, Math.max(topK * 5, 50)); + const filtered = raw.filter(h => { + if (p.severity && h.severity !== p.severity) return false; + if (p.ecosystem && !h.ecosystems.some(e => ecosystemMatches(e, p.ecosystem!))) return false; + if (cweReq && !cweFilterMatches(h.cweIds, cweReq)) return false; + if (p.web_app_only && !isWebAppAdvisory(h.cweIds)) return false; + return true; + }).slice(0, topK); + + const results = filtered.map(h => ({ + ghsa_id: h.ghsa_id, + cve_id: h.cve_id || undefined, + summary: h.summary, + severity: h.severity, + published_at: h.published ? new Date(h.published).toISOString() : undefined, + cwes: h.cweIds, + packages: h.packages.slice(0, 8), + ecosystems: [...new Set(h.ecosystems)], + score: Number(h.scores.final.toFixed(4)), + temporal_relevance: Number(h.scores.temporal.toFixed(2)), + url: `https://github.com/advisories/${h.ghsa_id}`, + })); + + return { + content: [{ + type: 'text', + text: JSON.stringify({ + query: p.query, + count: results.length, + index: { docs: index.meta.count, model: index.meta.model, db_commit: index.meta.dbCommit }, + results, + }, null, 2), + }], + }; +} From a4d88c7badaad7d97f1fb2a5bb93ff90d91cbd2c Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 20:34:02 -0700 Subject: [PATCH 09/19] docs(semantic): design note on weekly index redistribution (git-lfs, CI feasibility) Covers what/why to redistribute, git-lfs channel, cross-platform/ABI portability (LE + model pinning), and GitHub Actions scheduling feasibility (full rebuild ~borderline in 30m; incremental = seconds). --- docs/semantic-index-distribution.md | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/semantic-index-distribution.md diff --git a/docs/semantic-index-distribution.md b/docs/semantic-index-distribution.md new file mode 100644 index 0000000..91a2b59 --- /dev/null +++ b/docs/semantic-index-distribution.md @@ -0,0 +1,107 @@ +# Design note: weekly redistribution of the semantic index + +Status: prototype (`proto/semantic-search`). Applies to the local hybrid search +index in `src/semantic/` (see its README). + +## Why redistribute at all + +Building the reviewed-tier index (~35k advisories) is a one-off CPU cost +(~15–20 min at fp32 on a typical 4-core CPU) and needs the embedding model + +toolchain. The output is **portable data**, so the right pattern is *build once, +ship the artifact* — consumers read the vectors instead of recomputing them, and +only embed the query at runtime. + +## What is distributed + +`.semantic-index/` (~100 MB total at 35k): + +| File | ~Size @35k | Notes | +|---|---|---| +| `embeddings.bin` | ~52 MB | `count × 384 × 4` bytes, raw float32 | +| `bm25.json` | ~35–45 MB | lexical index (vocab + tf/df) | +| `docs.json` | ~13 MB | rerank/display fields | +| `meta.json` | tiny | model, dim, count, **dbCommit**, builtAt | + +Every blob is traceable to the advisory-database commit it was built from +(`meta.dbCommit`) and the model that produced it (`meta.model`). + +## Channel: git-lfs + +- Track the binary blob (and the large `bm25.json`) via `.gitattributes` LFS: + ``` + .semantic-index/embeddings.bin filter=lfs diff=lfs merge=lfs -text + .semantic-index/bm25.json filter=lfs diff=lfs merge=lfs -text + ``` +- Consumers `git lfs pull`; no re-embedding of the corpus, only query embedding. +- Alternatives: a GitHub **Release asset** (`index.tar.gz`) or an OCI artifact — + LFS is simplest for "the index lives with the repo," a Release is better if you + don't want LFS bandwidth on every clone. + +## Cross-platform / ABI (portable — with two guards) + +- `embeddings.bin` is native-endian float32. All real targets (Windows/Linux/mac, + x86-64 **and** ARM64) are little-endian and IEEE-754, so bytes round-trip + everywhere. Harden by writing/reading with an explicit little-endian `DataView` + and stamping a `formatVersion`. +- Consumers must embed **queries** with the *same model* the blob was built with, + or the query lands in a different vector space. Pin the model **revision** and, + on load, assert `meta.model === runtimeModel` and `meta.dim === EMBED_DIM`. +- `onnxruntime-node` ships per-platform prebuilt binaries — an install concern for + query embedding, not a data concern; the blob itself is arch-neutral. +- Re-embedding is **not** bit-identical across arch/EP/thread-count (last-few-ULP), + but distributing the prebuilt blob sidesteps that entirely; keep one canonical + builder platform for reproducible artifacts. + +## Weekly schedule on GitHub Actions — can it run in 30 min? + +Platform limits are 6 h/job (hosted) / 5 days (self-hosted), so **30 min is a +budget, not a limit**. Budget for a *full* rebuild on `ubuntu-latest` (4 vCPU): + +| Step | ~Time | +|---|---| +| checkout (+lfs) | ~30 s | +| advisory-db shallow clone | ~1–3 min (cacheable) | +| npm ci + build | ~1–2 min (cacheable) | +| model fetch | ~30 s (cacheable) | +| embed 35k (fp32, 4 vCPU) | ~15–25 min | +| git-lfs push (~100 MB) | ~1–2 min | +| **total** | **~20–33 min** | + +So a full rebuild is **borderline** on the smallest runner. It fits 30 min +comfortably with any of: + +1. **Incremental (recommended steady state):** only re-embed advisories changed + since the previous `meta.dbCommit` (`git diff --name-only ..HEAD`). Weekly + advisory churn is hundreds, not 35k → **seconds**, not minutes. Full rebuild + only when the model/format version changes. +2. **Bigger runner** (8/16 vCPU): ONNX scales near-linearly → embeddings ~5–10 min. +3. **Quantized model** (int8 MiniLM ONNX): ~2–4× faster embeddings. +4. **Cache** the DB clone + model weights (`actions/cache`) to remove startup cost. +5. **Shard** the corpus across a matrix and merge (for the initial full build). + +## Recommended pipeline (sketch) + +```yaml +on: + schedule: [{ cron: "0 6 * * 1" }] # weekly + workflow_dispatch: {} +jobs: + build-index: + runs-on: ubuntu-latest # or a larger runner for full rebuilds + steps: + - uses: actions/checkout@v7 # with: lfs: true + - restore caches: advisory-db, model weights, npm + - shallow update external/advisory-database + - npm ci && npm run build + - node dist/semantic/build-index.js --since # incremental (future flag) + - assert meta.model/dim; commit .semantic-index via git-lfs (or upload a Release asset) +``` + +## Caveats + +- **Size/bandwidth:** ~100 MB/week; LFS stores whole objects per version, so gzip + the blob and prefer incremental re-embedding to minimize churn. +- **Freshness vs cost:** weekly cadence balances advisory-database churn against + rebuild cost; bump to daily only with incremental builds. +- **Canonical builder:** pin the runner OS/arch + model revision so the published + artifact is reproducible; record all of it in `meta.json`. From d5b76a94aefb2d2bada69e60063db39c72f45a55 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 20:45:52 -0700 Subject: [PATCH 10/19] =?UTF-8?q?docs(semantic):=20detailed=20design=20?= =?UTF-8?q?=E2=80=94=20tool,=20measured=20timing,=20git-lfs=20distribution?= =?UTF-8?q?,=20weekly=20refresh,=20sparse=20checkout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers the semantic_search tool + pipeline, empirical build/size numbers (35k in ~22m local; index ~96MB), git-lfs on a dedicated semantic-index branch, weekly scheduled refresh, and sparse-checkout/partial-clone recipes to avoid clone bloat. --- docs/semantic-search-design.md | 212 +++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 docs/semantic-search-design.md diff --git a/docs/semantic-search-design.md b/docs/semantic-search-design.md new file mode 100644 index 0000000..91777f2 --- /dev/null +++ b/docs/semantic-search-design.md @@ -0,0 +1,212 @@ +# Semantic search — design & distribution + +Status: prototype (`proto/semantic-search`). Local-only hybrid search over GitHub +Security Advisories. No external engine or service; all inference on-device. + +Companion docs: [`src/semantic/README.md`](../src/semantic/README.md) (how to run), +[`semantic-index-distribution.md`](semantic-index-distribution.md) (CI/portability deep-dive). + +--- + +## 1. The tool: `semantic_search` + +An MCP tool registered in `createAdvisoryServer()` (so it's on both the stdio and +HTTP servers), alongside `list_advisories` / `get_advisory`. + +Input: + +| field | type | notes | +|---|---|---| +| `query` | string (required) | natural language; may contain a period ("SSRF in August 2026", "recent RCE") | +| `top_k` | number | default 10, max 50 | +| `web_app_only` | bool | keep only web-app CWE classes | +| `severity` | enum | post-filter | +| `ecosystem` | enum | post-filter (mapped to OSV names) | +| `cwes` | string | comma-separated, bare or `CWE-` prefixed | + +Output: JSON `{ query, count, index:{docs,model,db_commit}, results:[…] }` where each +result carries `ghsa_id, cve_id, summary, severity, published_at, cwes, packages, +ecosystems, score, temporal_relevance, url`. If the index isn't built, the tool +returns build instructions (non-fatal). + +### Retrieval pipeline + +```mermaid +flowchart LR + Q[query] --> T{temporal intent?} + T -- strip period --> R[residual text] + R --> E[query embedding] + R --> L[BM25] + E --> V[vector kNN] + V --> F[RRF fusion] + L --> F + F --> RR[field-aware rerank + + temporal proximity] + T -- period --> RR + RR --> K[top_k + post-filters] +``` + +1. **Temporal parse** — a period is lifted out of the query; recall runs on the + *residual* text so date words don't pollute matching. +2. **Recall** — dense vector kNN (cosine over unit vectors) ∪ BM25 lexical. +3. **Fuse** — Reciprocal Rank Fusion (`k=60`). +4. **Rerank** — field-aware boosts (exact GHSA/CVE/package/CWE, summary phrase + overlap) + temporal proximity to the requested period (in-window 1.0, exp + decay, half-life 45 d). A local cross-encoder can replace the field reranker + behind the same interface later. +5. **Post-filter** — `web_app_only` / `severity` / `ecosystem` / `cwes`, then `top_k`. + +### Modules (`src/semantic/`) + +| file | role | +|---|---| +| `document.ts` | Advisory → doc (embedded text + rerank fields + `published`) | +| `embeddings.ts` | Local ONNX embeddings (`@huggingface/transformers`, MiniLM 384-dim), offline | +| `bm25.ts` | Compact Okapi BM25, identifier-preserving tokenizer | +| `temporal.ts` | Parse period from query + publish-date proximity score | +| `store.ts` | Persist/load `embeddings.bin` + `bm25.json` + `docs.json` + `meta.json` | +| `hybrid.ts` | Recall → RRF → rerank | +| `build-index.ts` / `query.ts` | CLIs | +| `../tools/semantic-search.ts` | MCP tool wrapper | + +--- + +## 2. Timing & size (measured) + +Corpus: **35,514** `github-reviewed` advisories. Model: `Xenova/all-MiniLM-L6-v2` +(384-dim, fp32, CPU). + +| Metric | Value | +|---|---| +| Full index build (local, CPU) | **1,335 s ≈ 22 min** (35,514 docs) | +| CI build step (2,000-doc test, incl. 35k DB scan + model download) | 118 s; total job **3.5 min** | +| Extrapolated full build on `ubuntu-latest` (4 vCPU) | **~25–40 min** (well within a 2 h budget) | +| Query latency (35k, brute-force cosine + BM25 + rerank) | sub-second | +| `embeddings.bin` | **52.0 MB** (`35,514 × 384 × 4`) | +| `bm25.json` | 31.9 MB | +| `docs.json` | 11.6 MB | +| **index total** | **~96 MB** | + +Speed-ups if needed: larger runner (ONNX scales ~linearly), int8-quantized model +(2–4×), incremental re-embed of only changed advisories. + +--- + +## 3. Distribution model — GitHub git-lfs + +Building takes ~22 min and needs the model + toolchain; the output is portable +data, so **build once, distribute the artifact**. Consumers read the vectors and +only embed the *query* at runtime. + +### Where to store it — a dedicated `semantic-index` branch (recommended) + +Keeping ~96 MB (growing weekly) on `main` bloats every clone's history. Instead, +publish the index to an **orphan branch** so `main` stays lean and only those who +want the index fetch it: + +```bash +git checkout --orphan semantic-index +git rm -rf . # empty tree +# place .semantic-index/* here (or at repo root), then: +git lfs track "embeddings.bin" "bm25.json" "docs.json" +git add .gitattributes embeddings.bin bm25.json docs.json meta.json +git commit -m "semantic index @ " +git push origin semantic-index +``` + +`.gitattributes` (LFS pointers, so the branch tree stays tiny): + +``` +embeddings.bin filter=lfs diff=lfs merge=lfs -text +bm25.json filter=lfs diff=lfs merge=lfs -text +docs.json filter=lfs diff=lfs merge=lfs -text +``` + +Alternative with zero clone-bloat: publish `index.tar.gz` as a **GitHub Release +asset** (downloaded on demand, never part of `git clone`). Use LFS if you want the +index versioned with the repo; use Releases if you only ever want "latest". + +### Portability (see distribution note for detail) + +- `embeddings.bin` is little-endian float32 → portable across Windows/Linux/macOS + on x86-64 and ARM64. Harden with explicit LE serialization + a `formatVersion`. +- Consumers must embed queries with the **same model** — pin the model revision; + assert `meta.model` / `meta.dim` on load. + +--- + +## 4. Weekly refresh + +Scheduled GitHub Actions job (must live on the default branch to fire on +`schedule`). Prototype workflow: [`.github/workflows/semantic-index.yml`](../.github/workflows/semantic-index.yml) +(currently uploads an artifact; the commit-back-via-LFS step is sketched in +comments and needs `permissions: contents: write`). + +Flow: + +1. `schedule: cron "0 6 * * 1"` (+ `workflow_dispatch`). +2. Shallow-update `external/advisory-database`; read its `HEAD` (`meta.dbCommit`). +3. **Incremental** (steady state): re-embed only advisories changed since the + previous `meta.dbCommit` (`git diff --name-only ..HEAD`). Weekly churn is + hundreds → seconds. Full rebuild only when the model/format version changes. +4. Write the index, assert `meta`, commit to the `semantic-index` branch via LFS + (or upload a Release asset). Record `dbCommit` + `builtAt` + model revision. + +Storage growth: each weekly blob is a new LFS object (~96 MB) and old versions are +retained → ~5 GB/yr of LFS storage. Mitigate by: the orphan branch (keeps it off +`main`), periodic history squash on that branch, `git lfs prune`, incremental +builds (smaller deltas), or Release assets with retention. + +--- + +## 5. Consuming without the bloat — sparse checkout & partial clone + +The index should never force itself onto people who don't need it. + +**A. Normal dev clone that skips the index blobs** (get pointers, not 96 MB): + +```bash +GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/microsoft/github-advisory-mcp +# or persist it: +git config lfs.fetchexclude ".semantic-index/**" +``` + +**B. Fetch ONLY the index, minimal footprint** (partial clone + sparse-checkout + +targeted LFS pull): + +```bash +git clone --filter=blob:none --no-checkout \ + https://github.com/microsoft/github-advisory-mcp idx +cd idx +git sparse-checkout init --cone +git sparse-checkout set .semantic-index # only this path +git checkout semantic-index # the index branch +git lfs pull --include=".semantic-index/**" # pull just these LFS objects +``` + +**C. Exclude the index from a working checkout** (cone mode, everything *but* the +index — use a non-cone negative pattern): + +```bash +git sparse-checkout init --no-cone +printf '/*\n!/.semantic-index/\n' > .git/info/sparse-checkout +git read-tree -mu HEAD +``` + +**D. CI/consumer that only needs the latest artifact** — skip git entirely and +download the Release asset (`gh release download` / `curl`), or the Actions +artifact. + +> Rule of thumb: contributors use **A** (no index), agents/consumers use **B** +> (index only), and automation uses **D** (asset download). `main` never carries +> the blobs, so a default clone stays small regardless. + +--- + +## 6. Roadmap + +- Pin the model **revision** + assert on load; explicit little-endian serialization. +- Incremental `--since ` build flag for the weekly job. +- Optional local cross-encoder reranker (`Xenova/bge-reranker-base`). +- ANN (hnswlib) + on-disk store (sqlite-vec) when adding the 370k unreviewed tier. +- Wire the temporal signal into the tool as a soft prefilter for large corpora. From bfee783b29639f156f0785a0d35cf2a68fe9e01e Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 20:57:37 -0700 Subject: [PATCH 11/19] test(semantic): unit tests + coverage checks + hygiene MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 82 unit tests across bm25, temporal, document, store (save/load round-trip incl. embeddings byte-exactness), and hybrid (RRF + field/temporal rerank via injected query embedding — no model needed) - Add @vitest/coverage-v8 + test:coverage script; per-file coverage thresholds for the deterministic semantic modules - Hygiene: store index dir resolved at runtime (indexDir()) not import time; hybridSearch accepts an injectable query embedding for tests; drop unused import --- package-lock.json | 229 ++++++++++++++++++++++++++++ package.json | 2 + src/semantic/build-index.ts | 5 +- src/semantic/hybrid.ts | 5 +- src/semantic/store.ts | 27 ++-- src/tools/semantic-search.ts | 4 +- test/unit/semantic-bm25.test.ts | 40 +++++ test/unit/semantic-document.test.ts | 61 ++++++++ test/unit/semantic-hybrid.test.ts | 53 +++++++ test/unit/semantic-store.test.ts | 67 ++++++++ test/unit/semantic-temporal.test.ts | 102 +++++++++++++ vitest.config.ts | 8 + 12 files changed, 585 insertions(+), 18 deletions(-) create mode 100644 test/unit/semantic-bm25.test.ts create mode 100644 test/unit/semantic-document.test.ts create mode 100644 test/unit/semantic-hybrid.test.ts create mode 100644 test/unit/semantic-store.test.ts create mode 100644 test/unit/semantic-temporal.test.ts diff --git a/package-lock.json b/package-lock.json index efc0d02..f66a0c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^26.4.1", + "@vitest/coverage-v8": "^4.1.11", "ai": "^7.0.93", "dotenv": "^17.4.2", "typescript": "^5.9.3", @@ -344,6 +345,66 @@ "node": ">=0.8.0" } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha1-fwhx2Zgk0jE31g+G/PYTD9WhtR8=", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha1-vYcITO0MeW7Ea9pJLeboPSnon8I=", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", + "integrity": "sha1-llNxai8Qxne5j7xj1L+wAMMCzxc=", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha1-Einu8x2FFW1w+j9M2Fk3bQ6vaGM=", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha1-u+EtyltO+YOg0K9LB7m8kOoKuro=", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@colors/colors": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", @@ -1374,6 +1435,16 @@ "node": ">=18.0.0" } }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha1-eg7mAfYPmaIMfHxf8MgDiMEYm9Y=", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", @@ -1381,6 +1452,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha1-2xXWeByTHzolGj2sOVAcmKYIL9A=", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@js-sdsl/ordered-map": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", @@ -3450,6 +3532,37 @@ "node": ">= 20" } }, + "node_modules/@vitest/coverage-v8": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.11.tgz", + "integrity": "sha1-bwY2q+fiPobdNRJyRFVL4sYLwqU=", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.1.11", + "ast-v8-to-istanbul": "^1.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.1.11", + "vitest": "4.1.11" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, "node_modules/@vitest/expect": { "version": "4.1.11", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", @@ -3677,6 +3790,18 @@ "node": ">=12" } }, + "node_modules/ast-v8-to-istanbul": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.5.tgz", + "integrity": "sha1-cIuutvXIeSJtESo0H/qCHEOIHS0=", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -4825,6 +4950,16 @@ "integrity": "sha1-4193ADU1sCl+oIVI9azmrbFIDdw=", "license": "ISC" }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha1-lEdx/ZyByBJlxNaUGGDaBrtZR5s=", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/has-property-descriptors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", @@ -4870,6 +5005,13 @@ "node": ">=16.9.0" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha1-39YAJ9o2o238viNiYsAKWCJoFFM=", + "dev": true, + "license": "MIT" + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -5055,6 +5197,45 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "license": "ISC" }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha1-LRZsSwZE1Do58Ev2wu3R5YXzF1Y=", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha1-kIMFusmlvRdaxqdEier9D8JEWn0=", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha1-y0U1FitXhKpiPO4hpyUs8sgHrJM=", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jackspeak": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", @@ -5079,6 +5260,13 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha1-3/51mbSou3/jCv+NAjUjTf+3mDE=", + "dev": true, + "license": "MIT" + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -5259,6 +5447,34 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.4.tgz", + "integrity": "sha1-u+ON/WA3ZwBX8zq/IriqedXpnQo=", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "source-map-js": "^1.2.1" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha1-w8IwencSd82WODBfkVwprnQbYU4=", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/matcher": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", @@ -6451,6 +6667,19 @@ "node": ">=8" } }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha1-G33NyzK4E4gBs+R4umpRyqiWSNo=", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/systeminformation": { "version": "5.33.8", "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.33.8.tgz", diff --git a/package.json b/package.json index 3ee2ce8..c8471a6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "start:stdio": "node dist/index.js", "start:http": "node dist/http-server.js", "test": "vitest run", + "test:coverage": "vitest run --coverage", "test:e2e": "vitest run test/e2e", "test:integration": "vitest run test/integration", "test:watch": "vitest --watch", @@ -63,6 +64,7 @@ "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^26.4.1", + "@vitest/coverage-v8": "^4.1.11", "ai": "^7.0.93", "dotenv": "^17.4.2", "typescript": "^5.9.3", diff --git a/src/semantic/build-index.ts b/src/semantic/build-index.ts index f1c6ce3..3f1b8e9 100644 --- a/src/semantic/build-index.ts +++ b/src/semantic/build-index.ts @@ -9,12 +9,11 @@ */ import { execFileSync } from 'child_process'; -import { join } from 'path'; import { LocalRepositoryDataSource } from '../datasources/local-repository.js'; import { toDoc, type AdvisoryDoc } from './document.js'; import { embedBatch, modelName, EMBED_DIM } from './embeddings.js'; import { Bm25 } from './bm25.js'; -import { saveIndex, INDEX_DIR, type IndexMeta } from './store.js'; +import { saveIndex, indexDir, type IndexMeta } from './store.js'; const REPO = process.env.ADVISORY_REPO_PATH || './external/advisory-database'; const BATCH = Number(process.env.SEMANTIC_BATCH || 64); @@ -51,7 +50,7 @@ async function main() { dbCommit: dbCommit(), builtAt: new Date().toISOString(), }; await saveIndex(meta, docs, vectors, bm25.toJSON()); - console.error(`[build] saved index -> ${join(INDEX_DIR)} (${docs.length} docs)`); + console.error(`[build] saved index -> ${indexDir()} (${docs.length} docs)`); } main().catch(err => { console.error(err); process.exit(1); }); diff --git a/src/semantic/hybrid.ts b/src/semantic/hybrid.ts index e1fc4e4..bf478fa 100644 --- a/src/semantic/hybrid.ts +++ b/src/semantic/hybrid.ts @@ -56,7 +56,8 @@ export async function hybridSearch( index: LoadedIndex, query: string, topK = 10, - fetch = 50 + fetch = 50, + queryEmbedding?: Float32Array // inject to search without loading the model (tests) ): Promise { const bm25 = Bm25.fromJSON(index.bm25); @@ -65,7 +66,7 @@ export async function hybridSearch( const temporal = parseTemporal(query); const recallQuery = temporal.present && temporal.residual ? temporal.residual : query; - const qEmbedding = await embed(recallQuery); + const qEmbedding = queryEmbedding ?? await embed(recallQuery); const qTokens = new Set(tokenize(recallQuery)); const vec = vectorTopK(qEmbedding, index.vectors, index.docs.length, fetch); diff --git a/src/semantic/store.ts b/src/semantic/store.ts index 81135ac..9104fbb 100644 --- a/src/semantic/store.ts +++ b/src/semantic/store.ts @@ -10,7 +10,9 @@ import { EMBED_DIM } from './embeddings.js'; import type { AdvisoryDoc } from './document.js'; import type { Bm25Data } from './bm25.js'; -export const INDEX_DIR = process.env.SEMANTIC_INDEX_DIR || './.semantic-index'; +export function indexDir(): string { + return process.env.SEMANTIC_INDEX_DIR || './.semantic-index'; +} export interface IndexMeta { model: string; @@ -36,26 +38,29 @@ export async function saveIndex( vectors: Float32Array[], bm25: Bm25Data ): Promise { - await mkdir(INDEX_DIR, { recursive: true }); + const dir = indexDir(); + await mkdir(dir, { recursive: true }); const flat = new Float32Array(vectors.length * EMBED_DIM); vectors.forEach((v, i) => flat.set(v, i * EMBED_DIM)); - await writeFile(join(INDEX_DIR, 'embeddings.bin'), Buffer.from(flat.buffer)); + await writeFile(join(dir, 'embeddings.bin'), Buffer.from(flat.buffer)); const stored: StoredDoc[] = docs.map(({ text, ...rest }) => rest); - await writeFile(join(INDEX_DIR, 'docs.json'), JSON.stringify(stored)); - await writeFile(join(INDEX_DIR, 'bm25.json'), JSON.stringify(bm25)); - await writeFile(join(INDEX_DIR, 'meta.json'), JSON.stringify(meta, null, 2)); + await writeFile(join(dir, 'docs.json'), JSON.stringify(stored)); + await writeFile(join(dir, 'bm25.json'), JSON.stringify(bm25)); + await writeFile(join(dir, 'meta.json'), JSON.stringify(meta, null, 2)); } export function indexExists(): boolean { + const dir = indexDir(); return ['embeddings.bin', 'docs.json', 'bm25.json', 'meta.json'] - .every(f => existsSync(join(INDEX_DIR, f))); + .every(f => existsSync(join(dir, f))); } export async function loadIndex(): Promise { - const meta: IndexMeta = JSON.parse(await readFile(join(INDEX_DIR, 'meta.json'), 'utf-8')); - const docs: StoredDoc[] = JSON.parse(await readFile(join(INDEX_DIR, 'docs.json'), 'utf-8')); - const bm25: Bm25Data = JSON.parse(await readFile(join(INDEX_DIR, 'bm25.json'), 'utf-8')); - const buf = await readFile(join(INDEX_DIR, 'embeddings.bin')); + const dir = indexDir(); + const meta: IndexMeta = JSON.parse(await readFile(join(dir, 'meta.json'), 'utf-8')); + const docs: StoredDoc[] = JSON.parse(await readFile(join(dir, 'docs.json'), 'utf-8')); + const bm25: Bm25Data = JSON.parse(await readFile(join(dir, 'bm25.json'), 'utf-8')); + const buf = await readFile(join(dir, 'embeddings.bin')); const vectors = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4); return { meta, docs, vectors, bm25 }; } diff --git a/src/tools/semantic-search.ts b/src/tools/semantic-search.ts index 71a1533..5fe36e1 100644 --- a/src/tools/semantic-search.ts +++ b/src/tools/semantic-search.ts @@ -6,7 +6,7 @@ import { z } from 'zod'; import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; import { createLogger } from '../logger.js'; -import { indexExists, loadIndex, INDEX_DIR, type LoadedIndex } from '../semantic/store.js'; +import { indexExists, loadIndex, indexDir, type LoadedIndex } from '../semantic/store.js'; import { hybridSearch } from '../semantic/hybrid.js'; import { ecosystemMatches, cweFilterMatches, isWebAppAdvisory } from '../datasources/local-repository.js'; @@ -40,7 +40,7 @@ export async function semanticSearch(params: unknown): Promise { isError: true, content: [{ type: 'text', - text: `Semantic index not found at ${INDEX_DIR}. Build it first:\n` + + text: `Semantic index not found at ${indexDir()}. Build it first:\n` + ` ADVISORY_REPO_PATH=./external/advisory-database \\\n` + ` SEMANTIC_MODEL_CACHE= \\\n` + ` node dist/semantic/build-index.js --limit 5000`, diff --git a/test/unit/semantic-bm25.test.ts b/test/unit/semantic-bm25.test.ts new file mode 100644 index 0000000..384d5ac --- /dev/null +++ b/test/unit/semantic-bm25.test.ts @@ -0,0 +1,40 @@ +import { describe, it, expect } from "vitest"; +import { tokenize, Bm25 } from "../../src/semantic/bm25.js"; + +describe("tokenize", () => { + it("preserves security identifiers and package names", () => { + expect(tokenize("GHSA-abcd-1234-xyz9 CVE-2026-1 next.js")).toEqual([ + "ghsa-abcd-1234-xyz9", + "cve-2026-1", + "next.js", + ]); + }); + it("lowercases, splits on punctuation, drops single chars", () => { + expect(tokenize("SQL Injection! a (blind)")).toEqual(["sql", "injection", "blind"]); + }); +}); + +describe("Bm25", () => { + const ids = ["a", "b", "c"]; + const texts = [ + "sql injection in the query builder", + "cross site scripting reflected xss", + "sql injection via sort parameter", + ]; + const bm = Bm25.build(ids, texts); + + it("ranks documents containing the query terms", () => { + const r = bm.search("sql injection", 5); + const top = r.map(([d]) => d); + expect(top).toContain(0); + expect(top).toContain(2); + expect(top).not.toContain(1); // xss doc has neither term + }); + it("returns nothing for out-of-vocabulary queries", () => { + expect(bm.search("kubernetes", 5)).toEqual([]); + }); + it("survives a JSON round-trip", () => { + const again = Bm25.fromJSON(JSON.parse(JSON.stringify(bm.toJSON()))); + expect(again.search("sql injection", 5)).toEqual(bm.search("sql injection", 5)); + }); +}); diff --git a/test/unit/semantic-document.test.ts b/test/unit/semantic-document.test.ts new file mode 100644 index 0000000..a1c422a --- /dev/null +++ b/test/unit/semantic-document.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect } from "vitest"; +import { toDoc } from "../../src/semantic/document.js"; +import type { Advisory } from "../../src/types/data-source.js"; + +function advisory(over: Partial = {}): Advisory { + return { + ghsa_id: "GHSA-aaaa-bbbb-cccc", + cve_id: "CVE-2026-9999", + summary: "SQL Injection in Example ORM", + description: "x".repeat(1000), + severity: "High", + cwes: [{ cwe_id: "CWE-89", name: "CWE-89" }], + vulnerabilities: [ + { package: { ecosystem: "Packagist", name: "acme/orm" } }, + ], + published_at: "2026-06-15T00:00:00Z", + updated_at: "2026-06-16T00:00:00Z", + ...over, + } as unknown as Advisory; +} + +describe("toDoc", () => { + it("maps identifiers, cwes, packages, ecosystems (lowercased) and severity", () => { + const d = toDoc(advisory()); + expect(d.id).toBe("GHSA-aaaa-bbbb-cccc"); + expect(d.cve_id).toBe("CVE-2026-9999"); + expect(d.cweIds).toEqual(["CWE-89"]); + expect(d.packages).toEqual(["acme/orm"]); + expect(d.ecosystems).toEqual(["packagist"]); + expect(d.severity).toBe("high"); + }); + + it("stores published/updated as epoch ms", () => { + const d = toDoc(advisory()); + expect(d.published).toBe(Date.parse("2026-06-15T00:00:00Z")); + expect(d.updated).toBe(Date.parse("2026-06-16T00:00:00Z")); + }); + + it("embedded text includes summary + ids + package + cwe, details truncated", () => { + const d = toDoc(advisory()); + expect(d.text).toContain("SQL Injection in Example ORM"); + expect(d.text).toContain("GHSA-aaaa-bbbb-cccc"); + expect(d.text).toContain("acme/orm"); + expect(d.text).toContain("CWE-89"); + // details capped at 500 chars, so the 1000-char body is not fully present + expect(d.text).not.toContain("x".repeat(600)); + }); + + it("tolerates missing dates/cwes/packages", () => { + const d = toDoc(advisory({ published_at: undefined, cwes: [], vulnerabilities: [] } as any)); + expect(d.published).toBe(0); + expect(d.cweIds).toEqual([]); + expect(d.packages).toEqual([]); + }); + + it("falls back to first line of details when summary is absent", () => { + const d = toDoc(advisory({ summary: undefined, cve_id: undefined } as any)); + expect(d.cve_id).toBe(""); + expect(typeof d.text).toBe("string"); + }); +}); diff --git a/test/unit/semantic-hybrid.test.ts b/test/unit/semantic-hybrid.test.ts new file mode 100644 index 0000000..7624c2f --- /dev/null +++ b/test/unit/semantic-hybrid.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect } from "vitest"; +import { EMBED_DIM } from "../../src/semantic/embeddings.js"; +import { hybridSearch } from "../../src/semantic/hybrid.js"; +import { Bm25 } from "../../src/semantic/bm25.js"; +import type { LoadedIndex, StoredDoc } from "../../src/semantic/store.js"; + +function sdoc(ghsa: string, summary: string, cwe: string, pkg: string, publishedYear: number): StoredDoc { + return { + id: ghsa, ghsa_id: ghsa, cve_id: "", summary, cweIds: [cwe], packages: [pkg], + ecosystems: ["npm"], severity: "high", published: Date.UTC(publishedYear, 0, 15), updated: 0, + }; +} + +// dim0-aligned unit vector by weight +function row(out: Float32Array, i: number, dim0: number) { out[i * EMBED_DIM + 0] = dim0; out[i * EMBED_DIM + 1] = Math.sqrt(Math.max(0, 1 - dim0 * dim0)); } + +const docs = [ + sdoc("GHSA-AAAA", "sql injection in query builder", "CWE-89", "acme/orm", 2020), + sdoc("GHSA-BBBB", "cross site scripting xss", "CWE-79", "foo/web", 2026), + sdoc("GHSA-CCCC", "sql injection via sort parameter", "CWE-89", "bar/db", 2026), +]; +const vectors = new Float32Array(docs.length * EMBED_DIM); +row(vectors, 0, 1.0); // aligned with query +row(vectors, 1, 0.0); // orthogonal +row(vectors, 2, 0.7); // partly aligned +const bm25 = Bm25.build(docs.map(d => d.id), docs.map(d => `${d.summary} ${d.ghsa_id}`)).toJSON(); +const index: LoadedIndex = { + meta: { model: "test", dim: EMBED_DIM, count: docs.length, dbCommit: "x", builtAt: "" }, + docs, vectors, bm25, +}; +const queryVec = (() => { const v = new Float32Array(EMBED_DIM); v[0] = 1; return v; })(); + +describe("hybridSearch", () => { + it("ranks the semantically + lexically matching doc first", async () => { + const hits = await hybridSearch(index, "sql injection", 3, 10, queryVec); + expect(hits[0].ghsa_id).toBe("GHSA-AAAA"); + // xss doc has no vector/lexical overlap -> ranks last + expect(hits[hits.length - 1].ghsa_id).toBe("GHSA-BBBB"); + }); + + it("field-aware rerank lifts an exact GHSA-id mention", async () => { + const hits = await hybridSearch(index, "sql injection GHSA-CCCC", 3, 10, queryVec); + expect(hits[0].ghsa_id).toBe("GHSA-CCCC"); + expect(hits[0].scores.rerank).toBeGreaterThanOrEqual(5); + }); + + it("temporal proximity scores the in-window doc 1.0 and decays others", async () => { + const hits = await hybridSearch(index, "sql injection in 2026", 3, 10, queryVec); + const byId = Object.fromEntries(hits.map(h => [h.ghsa_id, h])); + expect(byId["GHSA-CCCC"].scores.temporal).toBe(1); // 2026, in window + expect(byId["GHSA-AAAA"].scores.temporal).toBeLessThan(1); // 2020, decayed + }); +}); diff --git a/test/unit/semantic-store.test.ts b/test/unit/semantic-store.test.ts new file mode 100644 index 0000000..e421712 --- /dev/null +++ b/test/unit/semantic-store.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { mkdtempSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { EMBED_DIM } from "../../src/semantic/embeddings.js"; +import { saveIndex, loadIndex, indexExists, type IndexMeta } from "../../src/semantic/store.js"; +import type { AdvisoryDoc } from "../../src/semantic/document.js"; +import { Bm25 } from "../../src/semantic/bm25.js"; + +let dir: string; + +function doc(id: string, over: Partial = {}): AdvisoryDoc { + return { + id, text: `${id} sql injection`, ghsa_id: id, cve_id: "", summary: "sql injection", + cweIds: ["CWE-89"], packages: ["acme/orm"], ecosystems: ["packagist"], + severity: "high", published: Date.UTC(2026, 5, 1), updated: Date.UTC(2026, 5, 2), ...over, + }; +} + +function vec(seed: number): Float32Array { + const v = new Float32Array(EMBED_DIM); + for (let i = 0; i < EMBED_DIM; i++) v[i] = Math.sin(seed + i) * 0.01; + return v; +} + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "semidx-")); + process.env.SEMANTIC_INDEX_DIR = dir; +}); +afterAll(() => { + delete process.env.SEMANTIC_INDEX_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +describe("store save/load round-trip", () => { + const docs = [doc("GHSA-1"), doc("GHSA-2")]; + const vectors = [vec(1), vec(2)]; + const bm25 = Bm25.build(docs.map(d => d.id), docs.map(d => d.text)).toJSON(); + const meta: IndexMeta = { model: "test-model", dim: EMBED_DIM, count: 2, dbCommit: "abc1234", builtAt: "2026-09-13T00:00:00Z" }; + + it("indexExists is false before a build", () => { + expect(indexExists()).toBe(false); + }); + + it("writes all four files", async () => { + await saveIndex(meta, docs, vectors, bm25); + expect(indexExists()).toBe(true); + }); + + it("reloads meta and docs (text stripped) faithfully", async () => { + const ix = await loadIndex(); + expect(ix.meta).toEqual(meta); + expect(ix.docs).toHaveLength(2); + expect((ix.docs[0] as any).text).toBeUndefined(); + expect(ix.docs[0].ghsa_id).toBe("GHSA-1"); + expect(ix.docs[0].published).toBe(Date.UTC(2026, 5, 1)); + }); + + it("reloads embeddings byte-exact (serialization portability)", async () => { + const ix = await loadIndex(); + expect(ix.vectors.length).toBe(2 * EMBED_DIM); + for (let i = 0; i < EMBED_DIM; i++) { + expect(ix.vectors[i]).toBe(vectors[0][i]); + expect(ix.vectors[EMBED_DIM + i]).toBe(vectors[1][i]); + } + }); +}); diff --git a/test/unit/semantic-temporal.test.ts b/test/unit/semantic-temporal.test.ts new file mode 100644 index 0000000..28e9797 --- /dev/null +++ b/test/unit/semantic-temporal.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect } from "vitest"; +import { parseTemporal, temporalRelevance } from "../../src/semantic/temporal.js"; + +const NOW = Date.UTC(2026, 8, 13); // 2026-09-13 + +describe("parseTemporal", () => { + it("no period -> not present, residual unchanged", () => { + const t = parseTemporal("sql injection", NOW); + expect(t.present).toBe(false); + expect(t.residual).toBe("sql injection"); + }); + + it("explicit range", () => { + const t = parseTemporal("ssrf 2026-01-01..2026-06-30", NOW); + expect(t.present).toBe(true); + expect(new Date(t.start).toISOString().slice(0, 10)).toBe("2026-01-01"); + expect(new Date(t.end).toISOString().slice(0, 10)).toBe("2026-07-01"); // end exclusive (+1 day) + expect(t.residual).toBe("ssrf"); + }); + + it("month name + year, strips the phrase", () => { + const t = parseTemporal("sql injection in August 2026", NOW); + expect(new Date(t.start).toISOString().slice(0, 7)).toBe("2026-08"); + expect(new Date(t.end).toISOString().slice(0, 7)).toBe("2026-09"); + expect(t.residual).toBe("sql injection in"); + }); + + it("relative: last 30 days", () => { + const t = parseTemporal("rce in the last 30 days", NOW); + expect(t.present).toBe(true); + expect(t.start).toBeLessThan(NOW); + expect(NOW - t.start).toBeGreaterThanOrEqual(30 * 86_400_000 - 1); + expect(t.residual).toBe("rce in the"); + }); + + it("fuzzy 'recent' -> ~last 90 days", () => { + const t = parseTemporal("recent account takeover", NOW); + expect(t.present).toBe(true); + expect(Math.round((NOW - t.start) / 86_400_000)).toBe(90); + expect(t.residual).toBe("account takeover"); + }); + + it("bare year", () => { + const t = parseTemporal("xss 2024", NOW); + expect(new Date(t.start).toISOString().slice(0, 10)).toBe("2024-01-01"); + expect(new Date(t.end).toISOString().slice(0, 10)).toBe("2025-01-01"); + }); + + it("single ISO date -> that day", () => { + const t = parseTemporal("rce 2026-03-05", NOW); + expect(new Date(t.start).toISOString().slice(0, 10)).toBe("2026-03-05"); + expect(new Date(t.end).toISOString().slice(0, 10)).toBe("2026-03-06"); + expect(t.residual).toBe("rce"); + }); + + it("year-month -> whole month", () => { + const t = parseTemporal("idor 2026-02", NOW); + expect(new Date(t.start).toISOString().slice(0, 7)).toBe("2026-02"); + expect(new Date(t.end).toISOString().slice(0, 7)).toBe("2026-03"); + }); + + it("this month -> start of month .. now", () => { + const t = parseTemporal("ssrf this month", NOW); + expect(new Date(t.start).toISOString().slice(0, 10)).toBe("2026-09-01"); + expect(t.end).toBeGreaterThan(NOW - 1); + expect(t.residual).toBe("ssrf"); + }); + + it("last year -> previous calendar year", () => { + const t = parseTemporal("xxe last year", NOW); + expect(new Date(t.start).toISOString().slice(0, 10)).toBe("2025-01-01"); + expect(new Date(t.end).toISOString().slice(0, 10)).toBe("2026-01-01"); + }); + + it("today / yesterday", () => { + const today = parseTemporal("today", NOW); + expect(today.present).toBe(true); + const y = parseTemporal("yesterday", NOW); + expect(y.end).toBeLessThanOrEqual(today.start + 1); + }); +}); + +describe("temporalRelevance", () => { + const t = parseTemporal("2026-06-01..2026-06-30", NOW); + it("in-window scores 1.0", () => { + expect(temporalRelevance(Date.UTC(2026, 5, 15), t)).toBe(1); + }); + it("outside window decays between 0 and 1", () => { + const s = temporalRelevance(Date.UTC(2026, 2, 1), t); // ~3 months before + expect(s).toBeGreaterThan(0); + expect(s).toBeLessThan(1); + }); + it("closer dates score higher than farther ones", () => { + const near = temporalRelevance(Date.UTC(2026, 6, 10), t); + const far = temporalRelevance(Date.UTC(2025, 0, 1), t); + expect(near).toBeGreaterThan(far); + }); + it("unknown date or no period -> 0", () => { + expect(temporalRelevance(0, t)).toBe(0); + expect(temporalRelevance(Date.UTC(2026, 5, 15), parseTemporal("no date", NOW))).toBe(0); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 511b276..406da0e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -16,6 +16,7 @@ export default defineConfig({ coverage: { provider: "v8", reporter: ["text", "json", "html"], + include: ["src/**/*.ts"], exclude: [ "node_modules/", "dist/", @@ -23,6 +24,13 @@ export default defineConfig({ "**/*.spec.ts", "external/", ], + // Enforce coverage on the deterministic semantic modules that have tests. + thresholds: { + "src/semantic/bm25.ts": { lines: 85, functions: 85, statements: 85, branches: 70 }, + "src/semantic/temporal.ts": { lines: 80, functions: 90, statements: 80, branches: 70 }, + "src/semantic/document.ts": { lines: 90, functions: 90, statements: 90, branches: 65 }, + "src/semantic/store.ts": { lines: 85, functions: 85, statements: 85, branches: 50 }, + }, }, }, resolve: { From c4057c9622c72835e90529d594f044a0d0f5fdaf Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 20:58:09 -0700 Subject: [PATCH 12/19] ci: run unit tests with coverage (enforces semantic thresholds) --- .github/workflows/build.yml | 2 +- src/semantic/README.md | 5 +++-- src/semantic/build-index.ts | 2 +- src/semantic/query.ts | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 48ac640..69082d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: run: ls -lh dist/ - name: Run unit tests - run: npx vitest run test/unit + run: npx vitest run test/unit --coverage e2e: runs-on: ubuntu-latest diff --git a/src/semantic/README.md b/src/semantic/README.md index 4394450..235e56b 100644 --- a/src/semantic/README.md +++ b/src/semantic/README.md @@ -24,8 +24,9 @@ memory store. ## Run ```powershell -# reuse an existing transformers cache to stay fully offline (no HF download) -$env:SEMANTIC_MODEL_CACHE = "C:\build\romulus-gym\services\memory\models\.cache" +# point at a pre-downloaded @huggingface/transformers cache to stay offline +# (omit it + set SEMANTIC_ALLOW_REMOTE=true to download the model instead) +$env:SEMANTIC_MODEL_CACHE = "" $env:ADVISORY_REPO_PATH = "./external/advisory-database" npm run build diff --git a/src/semantic/build-index.ts b/src/semantic/build-index.ts index 3f1b8e9..8a59843 100644 --- a/src/semantic/build-index.ts +++ b/src/semantic/build-index.ts @@ -2,7 +2,7 @@ * Prototype CLI: build the local hybrid index from the advisory DB. * * ADVISORY_REPO_PATH=./external/advisory-database \ - * SEMANTIC_MODEL_CACHE=C:/build/romulus-gym/services/memory/models/.cache \ + * SEMANTIC_MODEL_CACHE= \ * node dist/semantic/build-index.js --limit 5000 * * --limit N build over the first N advisories (fast validation); omit for all reviewed. diff --git a/src/semantic/query.ts b/src/semantic/query.ts index b6f5bb0..dfb6bff 100644 --- a/src/semantic/query.ts +++ b/src/semantic/query.ts @@ -1,7 +1,7 @@ /** * Prototype CLI: query the local hybrid index. * - * SEMANTIC_MODEL_CACHE=C:/build/romulus-gym/services/memory/models/.cache \ + * SEMANTIC_MODEL_CACHE= \ * node dist/semantic/query.js "blind ORM injection via sort parameter" --top 8 */ From 1594c14adb945d2cfb37fe5b1c1839e8d0f4b9e9 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Sun, 13 Sep 2026 23:42:44 -0700 Subject: [PATCH 13/19] fix(deps): keep package.json in sync with lockfile (feed-available versions) --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index c8e6c99..c8471a6 100644 --- a/package.json +++ b/package.json @@ -56,16 +56,16 @@ "cors": "^2.8.6", "express": "^5.0.1", "winston": "^3.18.3", - "zod": "^4.6.2" + "zod": "^4.5.4" }, "devDependencies": { - "@ai-sdk/azure": "^4.0.68", + "@ai-sdk/azure": "^4.0.63", "@azure/identity": "^4.13.2", "@types/cors": "^2.8.17", "@types/express": "^5.0.0", - "@types/node": "^26.5.1", + "@types/node": "^26.4.1", "@vitest/coverage-v8": "^4.1.11", - "ai": "^7.0.97", + "ai": "^7.0.93", "dotenv": "^17.4.2", "typescript": "^5.9.3", "vitest": "^4.1.11" From a3986d8a81757d3890e637206224e2816fbfafdf Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Mon, 14 Sep 2026 00:07:49 -0700 Subject: [PATCH 14/19] docs: add AGENTS.md and refresh stale docs (hygiene) - Add root AGENTS.md: build/bootstrap/run/test contract for agents (generic, no internal infra) - CONTRIBUTING: Node 20+, branch from main (no dev branch), unit-test command - README: correct advisory counts (~370K/~35K), CI Node matrix 20.x/22.x, main-only triggers, drop stale CI notes --- AGENTS.md | 117 ++++++++++++++++++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 9 ++-- README.md | 36 +++++---------- 3 files changed, 134 insertions(+), 28 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..1236252 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,117 @@ +# AGENTS.md + +Guidance for AI coding agents (and humans) working in this repository. Keep changes +minimal, typed, and covered by tests. For contribution policy see +[CONTRIBUTING.md](CONTRIBUTING.md); for user-facing docs see [README.md](README.md). + +## What this is + +An MCP server that serves GitHub Security Advisories from a local clone of +`github/advisory-database`. Two-tier design: **MCP server (stdio or HTTP) → local +Express REST API → `LocalRepositoryDataSource`** (reads advisory JSON from disk). + +Core MCP tools: `list_advisories`, `get_advisory` (full parameters in the README). + +## Prerequisites + +- **Node.js 20+** (CI runs 20.x and 22.x) +- Git +- The server runs from `dist/` — **rebuild after every source change**. + +## Build + +```bash +npm install # or: npm ci (installs from package-lock.json) +npm run build # tsc -> dist/ +``` + +## Bootstrap the advisory database (needed to run/serve) + +The server reads advisory JSON from `ADVISORY_REPO_PATH` (default +`./external/advisory-database`). Clone it once: + +```bash +./scripts/setup-advisory-database.sh +# or manually: +git clone --depth=1 https://github.com/github/advisory-database.git external/advisory-database +``` + +`external/` is git-ignored. If absent, the server also auto-clones on first tool call. + +## Run + +stdio (what `.vscode/mcp.json` launches — for VS Code / agent use): + +```bash +ADVISORY_REPO_PATH=./external/advisory-database node dist/index.js +``` + +HTTP streaming: + +```bash +MCP_PORT=18006 ADVISORY_API_PORT=18005 \ +ADVISORY_REPO_PATH=./external/advisory-database node dist/http-server.js +# health: http://localhost:18006/health +``` + +## Test + +- **Unit (fast, hermetic — start here):** `npx vitest run test/unit` +- **E2E (spawns the server; needs the advisory DB):** `npm run test:e2e` + - The first `list_advisories` cold-builds an in-memory index over the whole DB; + if it flakes on the default 30s timeout, raise it: + `npx vitest run test/e2e --testTimeout=120000`. +- **Integration (Azure OpenAI):** `npm run test:integration` — requires Azure + credentials; skip unless you're specifically exercising the AI SDK path. +- `npm test` runs **all** suites (including integration). Prefer `test/unit` for + routine work. + +## Project layout + +| Path | Role | +|------|------| +| `src/index.ts` | stdio MCP entry (also starts the local REST API) | +| `src/http-server.ts` | HTTP streaming MCP entry + REST API | +| `src/local-server.ts` | local Express REST API the tools call | +| `src/server.ts` | shared MCP tool registration | +| `src/datasources/local-repository.ts` | advisory indexing + filters | +| `src/tools/` | MCP tool schemas/handlers | +| `test/{unit,e2e,integration}/` | test suites | + +## Conventions + +- TypeScript, ES modules, strict `tsc` build. Add/extend Zod schemas in `src/tools/` + for any new tool input — inputs are validated at the boundary. +- Rebuild (`npm run build`) and restart the server after edits — it runs from `dist/`. +- Don't commit `dist/`, `external/`, or machine-specific `.vscode/mcp.json` tweaks + (all git-ignored). + +## Key environment variables + +| Variable | Default | Purpose | +|----------|---------|---------| +| `ADVISORY_REPO_PATH` | `./external/advisory-database` | Advisory DB location | +| `MCP_PORT` | `18006` | HTTP MCP port | +| `ADVISORY_API_PORT` | `18005` | Local REST API port | +| `ADVISORY_API_HOST` | `127.0.0.1` | Local REST API host | +| `ADVISORY_REFRESH_ON_START` | `true` | `git pull` the DB on startup (`false` to skip) | + +## Semantic search (prototype) + +An optional `semantic_search` tool (local hybrid embeddings + BM25) lives under +`src/semantic/`. Build its index once, then it loads lazily on first call: + +```bash +npm run build +node dist/semantic/build-index.js --limit 5000 # omit --limit for the full reviewed corpus +``` + +Relevant env: `SEMANTIC_MODEL` (default `Xenova/all-MiniLM-L6-v2`), +`SEMANTIC_MODEL_CACHE` (point at a local transformers cache to run offline), +`SEMANTIC_ALLOW_REMOTE=true` (allow model download), `SEMANTIC_INDEX_DIR`. + +## Network note + +Installs use the public npm registry (`registry.npmjs.org`). If your environment +proxies or mirrors npm, configure a **git-ignored** `.npmrc` pointing at your mirror; +the committed `package-lock.json` keeps canonical public registry URLs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77705d3..5be330c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope - **OpenTelemetry** - Comprehensive instrumentation ### Prerequisites -- Node.js 18+ +- Node.js 20+ - Git - VS Code (recommended) @@ -62,8 +62,9 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope 5. **Run tests** ```bash - npm test + npx vitest run test/unit # fast, hermetic unit tests ``` + `npm test` runs every suite, including the Azure-credentialed integration tests. ### Submission Guidelines @@ -75,10 +76,10 @@ Before submitting an issue, please search the existing issues to avoid duplicate - Environment details (Node.js version, OS, etc.) #### Pull Requests -1. Create a new branch from `dev` +1. Create a new branch from `main` 2. Make your changes with clear, descriptive commit messages 3. Add tests for new functionality -4. Ensure all tests pass (`npm test`) +4. Ensure all tests pass (`npx vitest run test/unit`) 5. Update documentation as needed 6. Submit a pull request to the `dev` branch diff --git a/README.md b/README.md index 3c026fe..44d63ad 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ npm run build ``` 3. **Reload VS Code** - Copilot will automatically: - - Clone the advisory database (~310K advisories) on first use + - Clone the advisory database (~370K advisory files; ~35K github-reviewed served) on first use - Enable MCP tools: `list_advisories`, `get_advisory` 4. **Test in Copilot Chat:** @@ -292,7 +292,7 @@ All MCP tool parameters are validated using **Zod schemas**: **Considerations:** - **stdio mode**: Single-process, single-user - no rate limiting needed - **HTTP mode**: Consider adding rate limiting if exposed beyond localhost -- **Database queries**: Inherently rate-limited by disk I/O (310K+ files) +- **Database queries**: Inherently rate-limited by disk I/O (hundreds of thousands of files) **Future (HTTP mode):** ```typescript @@ -367,7 +367,7 @@ fatal: unable to access 'https://github.com/github/advisory-database.git/': Coul 3. Pre-download database: `git clone --depth=1 https://github.com/github/advisory-database.git external/advisory-database` 4. Point to existing database: `export ADVISORY_REPO_PATH=/path/to/existing/advisory-database` -**Timing:** Initial clone takes 2-5 minutes (310K+ files, ~500MB) +**Timing:** Initial clone takes 2-5 minutes (hundreds of thousands of files) ### Server Won't Start @@ -414,12 +414,12 @@ git clone --depth=1 https://github.com/github/advisory-database.git external/adv 1. First query always slower (loads database index into memory) 2. Use `per_page` parameter to limit results: `per_page: 10` 3. Filter by ecosystem to reduce search space: `ecosystem: "npm"` -4. Check disk I/O: Advisory database is 310K+ files +4. Check disk I/O: the advisory database is hundreds of thousands of files **Performance Benchmarks:** -- First query (cold start): 2-4 seconds (index load) -- Subsequent queries: 50-200ms (cached) -- Database size: ~500MB, 310,635 files +- First query (cold start): builds the in-memory index (slower) +- Subsequent queries: fast (index cached in memory) +- Database size: ~370K advisory JSON files (~35K github-reviewed served by default) ### Database Update Strategy @@ -442,7 +442,7 @@ git pull origin main - Use `Start.ps1` script for convenient startup ### Database Size -- The advisory-database is ~100K+ JSON files +- The advisory-database is ~370K advisory JSON files (~35K github-reviewed) - Shallow clone (`--depth=1`) recommended - First query loads entire index into memory (lazy loading) - Subsequent queries are fast (cached) @@ -453,27 +453,15 @@ git pull origin main **GitHub Actions Workflows:** -1. **Build Validation** (`.github/workflows/build.yml`) - - **Triggers:** Push to main/dev, PRs - - **Matrix:** Node.js 18.x, 20.x on Ubuntu latest - - **Steps:** Checkout → Setup Node → npm ci → Build → Verify artifacts - - **Timing:** ~27-33 seconds +1. **Build and Test** (`.github/workflows/build.yml`) + - **Triggers:** Push to `main`, PRs to `main` + - **Matrix:** Node.js 20.x, 22.x on Ubuntu latest + - **Steps:** Checkout → Setup Node → `npm ci` → build → verify artifacts → unit tests with coverage. End-to-end tests run on the `main` branch. 2. **Copilot PR Review** (`.github/workflows/copilot-review.yml`) - **Triggers:** PR opened or synchronized - **Action:** Automatically requests Copilot code review - **Permissions:** pull-requests: write, contents: read - - **Benefit:** Automated AI code review on every PR - -**Timing Estimates:** -- npm ci: ~10 seconds (dependency install) -- npm run build: ~4 seconds (TypeScript compilation) -- **Total CI time: ~27-33 seconds** - -**Note:** Tests are not run in CI (yet) because: -- Database clone takes 2-5 minutes (310K+ files) -- Would increase CI time to ~6-7 minutes per run -- Consider separate "full test" workflow for main branch only **Watch Mode:** ```powershell From 3abab034e5a2a36569c9033f09a9c2518e5656a4 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Mon, 14 Sep 2026 00:14:41 -0700 Subject: [PATCH 15/19] docs+chore: trim README, gate integration tests, tidy semantic docs - Move raw REST/MCP JSON-RPC recipes to docs/http-api.md and orchestrator/rate-limit examples to docs/integration.md; README 428->341 lines (M2) - Scope 'npm test' to unit; add test:all; skip Azure integration suite via describe.skipIf when AZURE_OPENAI_ENDPOINT unset (M3) - src/semantic/README.md: scope to how-to-run and link canonical docs/semantic-search-design.md; fix stale 'not wired in' note (M4) --- README.md | 110 +++----------------------- docs/http-api.md | 74 +++++++++++++++++ docs/integration.md | 43 ++++++++++ package.json | 5 +- src/semantic/README.md | 17 ++-- test/integration/ai-sdk-azure.test.ts | 3 +- 6 files changed, 143 insertions(+), 109 deletions(-) create mode 100644 docs/http-api.md create mode 100644 docs/integration.md diff --git a/README.md b/README.md index 44d63ad..2937b8a 100644 --- a/README.md +++ b/README.md @@ -129,10 +129,10 @@ Or query advisories directly: @workspace Get details for GHSA-jc85-fpwf-qm7x ``` -### Unit Tests (Automated) +### Automated Tests ```bash -npm test # All tests -npm run test:e2e # E2E tests (18 tests, ~9.5s after database cached) +npx vitest run test/unit # fast, hermetic unit tests (this is what `npm test` runs) +npm run test:e2e # end-to-end MCP tests (needs the advisory database) ``` ### Health Checks @@ -144,71 +144,10 @@ Invoke-RestMethod http://localhost:18006/health Invoke-RestMethod http://localhost:18005/health ``` -### Test Local REST API Directly +### Test the REST API and MCP protocol directly -**List advisories by ecosystem:** -```powershell -Invoke-RestMethod "http://localhost:18005/advisories?ecosystem=npm&per_page=5" -``` - -**Get specific advisory:** -```powershell -Invoke-RestMethod "http://localhost:18005/advisories/GHSA-jc85-fpwf-qm7x" -``` - -**Search advisories:** -```powershell -Invoke-RestMethod "http://localhost:18005/search?q=express" -``` - -### Test MCP Tools - -**Initialize Session:** -```powershell -$body = @{ - jsonrpc = "2.0" - id = 1 - method = "initialize" - params = @{ - protocolVersion = "2024-11-05" - capabilities = @{} - clientInfo = @{ name = "test-client"; version = "1.0.0" } - } -} | ConvertTo-Json -Depth 10 - -$response = Invoke-RestMethod -Uri "http://localhost:18006/mcp" -Method POST -Body $body -ContentType "application/json" -$sessionId = $response.result.sessionId -``` - -**List Tools:** -```powershell -$body = @{ - jsonrpc = "2.0" - id = 2 - method = "tools/list" -} | ConvertTo-Json - -Invoke-RestMethod -Uri "http://localhost:18006/mcp" -Method POST -Body $body -ContentType "application/json" -Headers @{"Mcp-Session-Id"=$sessionId} -``` - -**Call list_advisories:** -```powershell -$body = @{ - jsonrpc = "2.0" - id = 3 - method = "tools/call" - params = @{ - name = "list_advisories" - arguments = @{ - ecosystem = "npm" - severity = "high" - per_page = 5 - } - } -} | ConvertTo-Json -Depth 10 - -Invoke-RestMethod -Uri "http://localhost:18006/mcp" -Method POST -Body $body -ContentType "application/json" -Headers @{"Mcp-Session-Id"=$sessionId} -``` +Low-level REST and raw MCP JSON-RPC request recipes (health checks, session +initialize, `tools/list`, `tools/call`) live in [docs/http-api.md](docs/http-api.md). ## Environment Variables @@ -294,18 +233,8 @@ All MCP tool parameters are validated using **Zod schemas**: - **HTTP mode**: Consider adding rate limiting if exposed beyond localhost - **Database queries**: Inherently rate-limited by disk I/O (hundreds of thousands of files) -**Future (HTTP mode):** -```typescript -// Example: express-rate-limit for HTTP endpoints -import rateLimit from 'express-rate-limit'; - -const limiter = rateLimit({ - windowMs: 15 * 60 * 1000, // 15 minutes - max: 100 // limit each IP to 100 requests per windowMs -}); - -app.use('/mcp', limiter); -``` +**Future (HTTP mode):** add rate limiting if you expose the server beyond +localhost — see [docs/integration.md](docs/integration.md). ### Session Security @@ -325,27 +254,8 @@ app.use('/mcp', limiter); ## Integration with Orchestrator -The MCP Advisory server can be integrated with orchestration platforms: - -```python -from mcp import ClientSession -from mcp.client.stdio import stdio_client - -# Connect to MCP Advisory server -async with stdio_client( - command="node", - args=["dist/index.js"], - env={ - "ADVISORY_REPO_PATH": "/path/to/advisory-database" - } -) as (read, write): - async with ClientSession(read, write) as session: - # List npm advisories - result = await session.call_tool( - "list_advisories", - arguments={"ecosystem": "npm", "per_page": 10} - ) -``` +The stdio MCP server can be driven from any MCP client. A Python client example +and an HTTP rate-limiting snippet are in [docs/integration.md](docs/integration.md). ## Port Allocation diff --git a/docs/http-api.md b/docs/http-api.md new file mode 100644 index 0000000..7d22a1d --- /dev/null +++ b/docs/http-api.md @@ -0,0 +1,74 @@ +# Local HTTP / REST & MCP protocol recipes + +Low-level request examples for the HTTP server. For everyday use, prefer the MCP +tools via VS Code Copilot (see the [README](../README.md)); these recipes are for +directly exercising the REST API and the raw MCP JSON-RPC protocol. + +Start the HTTP server first: + +```powershell +$env:ADVISORY_REPO_PATH = "C:\path\to\advisory-database" +$env:MCP_PORT = "18006"; $env:ADVISORY_API_PORT = "18005" +node dist\http-server.js +``` + +## Health checks + +```powershell +Invoke-RestMethod http://localhost:18006/health # MCP server +Invoke-RestMethod http://localhost:18005/health # Local REST API +``` + +## Local REST API + +```powershell +# List advisories by ecosystem +Invoke-RestMethod "http://localhost:18005/advisories?ecosystem=npm&per_page=5" + +# Get a specific advisory +Invoke-RestMethod "http://localhost:18005/advisories/GHSA-jc85-fpwf-qm7x" + +# Search advisories +Invoke-RestMethod "http://localhost:18005/search?q=express" +``` + +## MCP tools over HTTP (raw JSON-RPC) + +**Initialize a session:** +```powershell +$body = @{ + jsonrpc = "2.0" + id = 1 + method = "initialize" + params = @{ + protocolVersion = "2024-11-05" + capabilities = @{} + clientInfo = @{ name = "test-client"; version = "1.0.0" } + } +} | ConvertTo-Json -Depth 10 + +$response = Invoke-RestMethod -Uri "http://localhost:18006/mcp" -Method POST -Body $body -ContentType "application/json" +$sessionId = $response.result.sessionId +``` + +**List tools:** +```powershell +$body = @{ jsonrpc = "2.0"; id = 2; method = "tools/list" } | ConvertTo-Json + +Invoke-RestMethod -Uri "http://localhost:18006/mcp" -Method POST -Body $body -ContentType "application/json" -Headers @{"Mcp-Session-Id"=$sessionId} +``` + +**Call `list_advisories`:** +```powershell +$body = @{ + jsonrpc = "2.0" + id = 3 + method = "tools/call" + params = @{ + name = "list_advisories" + arguments = @{ ecosystem = "npm"; severity = "high"; per_page = 5 } + } +} | ConvertTo-Json -Depth 10 + +Invoke-RestMethod -Uri "http://localhost:18006/mcp" -Method POST -Body $body -ContentType "application/json" -Headers @{"Mcp-Session-Id"=$sessionId} +``` diff --git a/docs/integration.md b/docs/integration.md new file mode 100644 index 0000000..b684987 --- /dev/null +++ b/docs/integration.md @@ -0,0 +1,43 @@ +# Integration & hardening examples + +Supplementary examples referenced from the [README](../README.md). These are +illustrative, not shipped code. + +## Embedding the MCP server in an orchestrator + +The stdio MCP server can be driven from any MCP client. Python example: + +```python +from mcp import ClientSession +from mcp.client.stdio import stdio_client + +# Connect to the MCP Advisory server +async with stdio_client( + command="node", + args=["dist/index.js"], + env={ + "ADVISORY_REPO_PATH": "/path/to/advisory-database" + } +) as (read, write): + async with ClientSession(read, write) as session: + result = await session.call_tool( + "list_advisories", + arguments={"ecosystem": "npm", "per_page": 10} + ) +``` + +## Rate limiting (HTTP mode) + +stdio mode is single-user and needs no rate limiting. If you expose the HTTP +server beyond localhost, add rate limiting — for example with `express-rate-limit`: + +```typescript +import rateLimit from 'express-rate-limit'; + +const limiter = rateLimit({ + windowMs: 15 * 60 * 1000, // 15 minutes + max: 100 // limit each IP to 100 requests per window +}); + +app.use('/mcp', limiter); +``` diff --git a/package.json b/package.json index c8471a6..c02de65 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,9 @@ "dev": "tsc --watch", "start:stdio": "node dist/index.js", "start:http": "node dist/http-server.js", - "test": "vitest run", - "test:coverage": "vitest run --coverage", + "test": "vitest run test/unit", + "test:all": "vitest run", + "test:coverage": "vitest run test/unit --coverage", "test:e2e": "vitest run test/e2e", "test:integration": "vitest run test/integration", "test:watch": "vitest --watch", diff --git a/src/semantic/README.md b/src/semantic/README.md index 235e56b..3e2e407 100644 --- a/src/semantic/README.md +++ b/src/semantic/README.md @@ -1,12 +1,17 @@ -# Semantic search prototype (local-only, advisory-specific) +# Semantic search — how to run (code layout) -> Prototype on branch `proto/semantic-search`. Not wired into the MCP tools yet. -> No external engine or service; runs entirely on-device. +> Prototype on branch `proto/semantic-search`. Runs entirely on-device (no external +> engine or service). +> +> **Design & rationale live in a separate doc:** +> [`docs/semantic-search-design.md`](../../docs/semantic-search-design.md). This file +> is the practical "build and run the index" guide next to the code. Hybrid retrieval for advisories: local embeddings + BM25, fused with Reciprocal -Rank Fusion, then a field-aware rerank. Advisory-specific (uses GHSA/CVE ids, -summary, CWEs, affected packages, ecosystem, severity) — not a general-purpose -memory store. +Rank Fusion, then a field-aware + temporal rerank. Exposed as the `semantic_search` +MCP tool (stdio + HTTP), alongside `list_advisories` / `get_advisory`. +Advisory-specific (GHSA/CVE ids, summary, CWEs, affected packages, ecosystem, +severity) — not a general-purpose memory store. ## Pieces diff --git a/test/integration/ai-sdk-azure.test.ts b/test/integration/ai-sdk-azure.test.ts index f231e7a..be9d449 100644 --- a/test/integration/ai-sdk-azure.test.ts +++ b/test/integration/ai-sdk-azure.test.ts @@ -212,7 +212,8 @@ function createAzureFetchAdapter(token: string): typeof fetch { }; } -describe("AI SDK Integration with Azure OpenAI (Azure AD Auth)", () => { +// Skips the whole suite unless Azure OpenAI is configured (no creds -> skipped, not failed). +describe.skipIf(!process.env.AZURE_OPENAI_ENDPOINT)("AI SDK Integration with Azure OpenAI (Azure AD Auth)", () => { const MCP_PORT = parseInt(process.env.MCP_PORT || "18006", 10); const API_PORT = parseInt(process.env.ADVISORY_API_PORT || "18005", 10); const REPO_PATH = From 3a49e7dddd1f0d4dda36ab227818b243aee23ee5 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Mon, 14 Sep 2026 00:27:41 -0700 Subject: [PATCH 16/19] fix(deps): force sharp>=0.35.4 to clear transitive advisories @huggingface/transformers pins a vulnerable sharp ^0.34.1; add an overrides entry (sharp>=0.35.4) to resolve GHSA-rgj7-g3m4-5g8c (libheif) and GHSA-f88m-g3jw-g9cj (libvips). npm audit: 0 vulnerabilities. Rationale documented in src/semantic/embeddings.ts (our usage is text-only). --- package-lock.json | 298 +++++++++++++++++++++---------------- package.json | 3 +- src/semantic/embeddings.ts | 5 + 3 files changed, 176 insertions(+), 130 deletions(-) diff --git a/package-lock.json b/package-lock.json index f66a0c7..acedf47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -951,9 +951,9 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha1-bgcy3K3hJrZnCveqFwYLkmg16oY=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", + "integrity": "sha1-vBCyYt4vyACIAT9fmYKY0ciQn78=", "cpu": [ "arm64" ], @@ -963,19 +963,19 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" + "@img/sharp-libvips-darwin-arm64": "1.3.3" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha1-Gbwd1uum1alig0mLnJ9AEYDunHs=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", + "integrity": "sha1-dsSf8E+z+dhGsNBXWEG37lm+7Gg=", "cpu": [ "x64" ], @@ -985,19 +985,38 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" + "@img/sharp-libvips-darwin-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", + "integrity": "sha1-ulX9YDxdHQGhyxT/tNlwvGztgOE=", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha1-KJTAy4fUInbDiJlC6OLbUXpJLEM=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", + "integrity": "sha1-CKbPT7TujUX5lASladyqayk5HZc=", "cpu": [ "arm64" ], @@ -1011,9 +1030,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha1-5jaB9FOalK+c0XJG7YiBc0OG+Mw=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", + "integrity": "sha1-FGHm+zEKhps8E1BFibjdXAZiTag=", "cpu": [ "x64" ], @@ -1027,9 +1046,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha1-uSYN0evm+eO9vL3KydKsEl81hS0=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", + "integrity": "sha1-cQMpQbn8v468PBtilJknuETLrWI=", "cpu": [ "arm" ], @@ -1043,9 +1062,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha1-sbKIs2hks7zlRa2R+m2tzxpK0xg=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", + "integrity": "sha1-y5OKOXG5o2MpvJlCfx5bcuJmui4=", "cpu": [ "arm64" ], @@ -1059,9 +1078,9 @@ } }, "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha1-S4Ps8qgpBXIis4hIx7Ai57TQeqc=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", + "integrity": "sha1-gkbS3sFSNP0cxUzFXcuhrrwmxBQ=", "cpu": [ "ppc64" ], @@ -1075,9 +1094,9 @@ } }, "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha1-iAtGeACeWiCArxkjMrALCq+KSN4=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", + "integrity": "sha1-Wf9mPCqmtBiFc1fNd8pMyuzpWHY=", "cpu": [ "riscv64" ], @@ -1091,9 +1110,9 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha1-dPNDyOEPrYIbOPdc7TBIiTncWew=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", + "integrity": "sha1-vBdFH4DZ8xZjs+kGFJ8Vmg7LHKg=", "cpu": [ "s390x" ], @@ -1107,9 +1126,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha1-30GD6L2EEPfWG2aFmjXt6rClMc4=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", + "integrity": "sha1-x9H4O8HP4iLNnckn88UTeql5HnY=", "cpu": [ "x64" ], @@ -1123,9 +1142,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha1-yNa0ghHfZxN1QQB+6NG3sfjKjgY=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", + "integrity": "sha1-MD3+Ko5tH9J56tVZ9NLM/yVF4qI=", "cpu": [ "arm64" ], @@ -1139,9 +1158,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha1-vhHHW+5bCAy+4xoVOod5RI+Rn3U=", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", + "integrity": "sha1-N1D3CtRq2H5KCO6NNyq5p+F+K2w=", "cpu": [ "x64" ], @@ -1155,9 +1174,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha1-X7DDaV3RJSLTnD/3pryBZGF4Cg0=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", + "integrity": "sha1-3o2g2i7lUiZ8rncVlzMqhjN45Ds=", "cpu": [ "arm" ], @@ -1167,19 +1186,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" + "@img/sharp-libvips-linux-arm": "1.3.3" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha1-eqd2TvnAAfFeYQVG1C/OVpEXkMw=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", + "integrity": "sha1-wvU7+RP6G0Z2KxksUnorNkeiEag=", "cpu": [ "arm64" ], @@ -1189,19 +1208,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" + "@img/sharp-libvips-linux-arm64": "1.3.3" } }, "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha1-nCE6gVIKIMr2aXjz1MB0Vv8uCBM=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", + "integrity": "sha1-xA18X4ziFsNSJMWO3O+OZEWTUlY=", "cpu": [ "ppc64" ], @@ -1211,19 +1230,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" + "@img/sharp-libvips-linux-ppc64": "1.3.3" } }, "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha1-zdKBgndOrb4E9iZ1oWqrvMuDP2A=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", + "integrity": "sha1-AyyiBucTsRee/g4Y1SUYjGcndzA=", "cpu": [ "riscv64" ], @@ -1233,19 +1252,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" + "@img/sharp-libvips-linux-riscv64": "1.3.3" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha1-k+rGAbnzKbsnkX4OGQmMci1jDfc=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", + "integrity": "sha1-GdrHUBoVyHSLx6ESGfyjkeQkttc=", "cpu": [ "s390x" ], @@ -1255,19 +1274,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" + "@img/sharp-libvips-linux-s390x": "1.3.3" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha1-VavHzXVP/KUAK2wrcZq9/IRoGag=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", + "integrity": "sha1-fR1BTUpTjgcEumt2zmVH5g9iEc4=", "cpu": [ "x64" ], @@ -1277,19 +1296,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" + "@img/sharp-libvips-linux-x64": "1.3.3" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha1-1lFe6XG7YvcwAaSCm52GWhG3cIY=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", + "integrity": "sha1-lTTJOAeCL0KC0HmjIvrMbbU9UPY=", "cpu": [ "arm64" ], @@ -1299,19 +1318,19 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha1-2Xl4rsfFIS+ZlxTy9bc2RX4S7p8=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", + "integrity": "sha1-9IjnoCkZkb/2Q874K1r9PP1w/PI=", "cpu": [ "x64" ], @@ -1321,38 +1340,54 @@ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + "@img/sharp-libvips-linuxmusl-x64": "1.3.3" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha1-LxWAOqYm+MWd18nQu8dm8atSz6A=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", + "integrity": "sha1-gfcfP8DYRH3Dr2yzeOJOdY/1QiU=", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", + "integrity": "sha1-d9LoiS06OCXw3JWrAs62E8shMoY=", "cpu": [ "wasm32" ], - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "license": "Apache-2.0", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.7.0" + "@img/sharp-wasm32": "0.35.4" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha1-Nwbp46w1/d/ByH+U6Enxt1MHzgo=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", + "integrity": "sha1-QQG0Tj+5fW8ac8KpxhQtSR0aqDE=", "cpu": [ "arm64" ], @@ -1362,16 +1397,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha1-C3EWZZmwSeAy8IX7kmPgL05HiN4=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", + "integrity": "sha1-jUdVvSbV/zT7iceQe9icCApdyfU=", "cpu": [ "ia32" ], @@ -1381,16 +1416,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": "^20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha1-qB/7AOaSZ80KHWJurtuKhDCysvg=", + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", + "integrity": "sha1-C2e9Cx8BEjqYKvzGug3AC4d7igo=", "cpu": [ "x64" ], @@ -1400,7 +1435,7 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" @@ -6359,47 +6394,52 @@ "license": "ISC" }, "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha1-tvFI5LjGHxeXveEanRz+u64sV7A=", - "hasInstallScript": true, + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", + "integrity": "sha1-Nh3zspWdrrOAVBKJlgRWxr4Pkt4=", "license": "Apache-2.0", "dependencies": { - "@img/colour": "^1.0.0", + "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", - "semver": "^7.7.3" + "semver": "^7.8.5" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" + "@img/sharp-darwin-arm64": "0.35.4", + "@img/sharp-darwin-x64": "0.35.4", + "@img/sharp-freebsd-wasm32": "0.35.4", + "@img/sharp-libvips-darwin-arm64": "1.3.3", + "@img/sharp-libvips-darwin-x64": "1.3.3", + "@img/sharp-libvips-linux-arm": "1.3.3", + "@img/sharp-libvips-linux-arm64": "1.3.3", + "@img/sharp-libvips-linux-ppc64": "1.3.3", + "@img/sharp-libvips-linux-riscv64": "1.3.3", + "@img/sharp-libvips-linux-s390x": "1.3.3", + "@img/sharp-libvips-linux-x64": "1.3.3", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", + "@img/sharp-libvips-linuxmusl-x64": "1.3.3", + "@img/sharp-linux-arm": "0.35.4", + "@img/sharp-linux-arm64": "0.35.4", + "@img/sharp-linux-ppc64": "0.35.4", + "@img/sharp-linux-riscv64": "0.35.4", + "@img/sharp-linux-s390x": "0.35.4", + "@img/sharp-linux-x64": "0.35.4", + "@img/sharp-linuxmusl-arm64": "0.35.4", + "@img/sharp-linuxmusl-x64": "0.35.4", + "@img/sharp-webcontainers-wasm32": "0.35.4", + "@img/sharp-win32-arm64": "0.35.4", + "@img/sharp-win32-ia32": "0.35.4", + "@img/sharp-win32-x64": "0.35.4" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/shebang-command": { diff --git a/package.json b/package.json index c02de65..21ac6e6 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,8 @@ "vitest": "^4.1.11" }, "overrides": { - "vite": "^7.1.12" + "vite": "^7.1.12", + "sharp": ">=0.35.4" }, "engines": { "node": ">=18.0.0" diff --git a/src/semantic/embeddings.ts b/src/semantic/embeddings.ts index 397d17e..9fcd94b 100644 --- a/src/semantic/embeddings.ts +++ b/src/semantic/embeddings.ts @@ -8,6 +8,11 @@ import { env, pipeline } from '@huggingface/transformers'; +// transformers pulls `sharp` (image codecs) transitively and pins a vulnerable +// `^0.34.1`. We only do TEXT embeddings, so sharp's libvips/libheif decode paths +// are never exercised here — but we still force sharp >= 0.35.4 via package.json +// `overrides` to clear GHSA-rgj7-g3m4-5g8c and GHSA-f88m-g3jw-g9cj. + const MODEL = process.env.SEMANTIC_MODEL || 'Xenova/all-MiniLM-L6-v2'; export const EMBED_DIM = Number(process.env.SEMANTIC_DIM || 384); From 0229d3c84accff04cd09334f4b23829b92de2fe5 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Mon, 14 Sep 2026 12:49:08 -0700 Subject: [PATCH 17/19] chore(deps): resolve to main's #85 versions (ai/zod/@types/node/@ai-sdk/azure) Lockfile regenerated on public npm (ai 7.0.99, zod 4.6.5, @types/node 26.5.1, @ai-sdk/azure 4.0.70), keeping sharp>=0.35.4 override + transformers/coverage-v8. Validated by CI build+unit tests; avoids downgrading main on merge. NOTE: this device's CFS proxy still lags these versions, so local npm ci needs the proxy to catch up; build/tests run on the already-installed modules. --- package-lock.json | 106 +++++++++++++++++++++++----------------------- package.json | 8 ++-- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/package-lock.json b/package-lock.json index acedf47..cf68645 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,19 +31,19 @@ "cors": "^2.8.6", "express": "^5.0.1", "winston": "^3.18.3", - "zod": "^4.5.4" + "zod": "^4.6.2" }, "bin": { "github-advisory-mcp": "dist/index.js" }, "devDependencies": { - "@ai-sdk/azure": "^4.0.63", + "@ai-sdk/azure": "^4.0.68", "@azure/identity": "^4.13.2", "@types/cors": "^2.8.17", "@types/express": "^5.0.0", - "@types/node": "^26.4.1", + "@types/node": "^26.5.1", "@vitest/coverage-v8": "^4.1.11", - "ai": "^7.0.93", + "ai": "^7.0.97", "dotenv": "^17.4.2", "typescript": "^5.9.3", "vitest": "^4.1.11" @@ -53,16 +53,16 @@ } }, "node_modules/@ai-sdk/azure": { - "version": "4.0.63", - "resolved": "https://registry.npmjs.org/@ai-sdk/azure/-/azure-4.0.63.tgz", - "integrity": "sha1-moa16CABP734MLIcM8YcxkdvWro=", + "version": "4.0.70", + "resolved": "https://registry.npmjs.org/@ai-sdk/azure/-/azure-4.0.70.tgz", + "integrity": "sha512-Z0G+LLgmNzPec/IaYRzEeHAT9riRrw3Ypme80ktuF636aMrIH9OD8lc+mXvi9QqpoYzZi+uWG+EyGVJF3Stp/w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@ai-sdk/deepseek": "3.0.39", - "@ai-sdk/openai": "4.0.60", - "@ai-sdk/provider": "4.0.10", - "@ai-sdk/provider-utils": "5.0.36" + "@ai-sdk/deepseek": "3.0.44", + "@ai-sdk/openai": "4.0.66", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" }, "engines": { "node": ">=22" @@ -72,14 +72,14 @@ } }, "node_modules/@ai-sdk/deepseek": { - "version": "3.0.39", - "resolved": "https://registry.npmjs.org/@ai-sdk/deepseek/-/deepseek-3.0.39.tgz", - "integrity": "sha1-/x/LRUDYPNnDbsgfVEjpMyQD3Rs=", + "version": "3.0.44", + "resolved": "https://registry.npmjs.org/@ai-sdk/deepseek/-/deepseek-3.0.44.tgz", + "integrity": "sha512-yhHGVJFpM5tdhhr33lE6Co8ziu8dNZxk7j5r3L3Pc04NMCTDmnXZ072FY4tqRDsRtIfBEgWqm/zPXNRvdHb44w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "4.0.10", - "@ai-sdk/provider-utils": "5.0.36" + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" }, "engines": { "node": ">=22" @@ -89,14 +89,14 @@ } }, "node_modules/@ai-sdk/gateway": { - "version": "4.0.75", - "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.75.tgz", - "integrity": "sha1-kR9taBuUu0otHKywWEhdYgNf6xk=", + "version": "4.0.80", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.80.tgz", + "integrity": "sha512-6t+07o8lSthpKf64Xb1qHWR2bWvJ3Fd2oFvS9fQc45p31bi2OUqan246e/ojAmZpWCiMPjKyQ4TBr4MYytnTiQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "4.0.10", - "@ai-sdk/provider-utils": "5.0.36", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", "@vercel/oidc": "3.2.0" }, "engines": { @@ -107,14 +107,14 @@ } }, "node_modules/@ai-sdk/openai": { - "version": "4.0.60", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-4.0.60.tgz", - "integrity": "sha1-kDQ7KAp3H5ON44MU4mkrm//hzrs=", + "version": "4.0.66", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-4.0.66.tgz", + "integrity": "sha512-W4GooZjROdOmf7dfjWlRWNVs+Vvr7ZwfDZbyAJRPo1bgQbpRoDd9ZcCwy/PX1X0s2G7B/dhN2RWOORsHtr0wGg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "4.0.10", - "@ai-sdk/provider-utils": "5.0.36" + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" }, "engines": { "node": ">=22" @@ -124,9 +124,9 @@ } }, "node_modules/@ai-sdk/provider": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.10.tgz", - "integrity": "sha1-HFHU4Jkd6ldzyKdPnaFsUlJ774w=", + "version": "4.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.14.tgz", + "integrity": "sha512-yukP2tbcQQErG5gLCMBGvpvb/rM3D3KlTKG6eKKOdNHLHqtNNDEeBxdYrY/JL+O76B2ig5dXY19H/f1HFSvRiQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -137,13 +137,13 @@ } }, "node_modules/@ai-sdk/provider-utils": { - "version": "5.0.36", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.36.tgz", - "integrity": "sha1-V+XFDvw3mPzk6BzSDH7JOyRKRmo=", + "version": "5.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.40.tgz", + "integrity": "sha512-zsXPwSAQ9mRJ2hvyITaLOYUyuGrBmzFhJXOg4mllGmla1PfNxZcm4GwqMiV2xQaDgcgBMdUGxXkp9xekZZNIkg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "4.0.10", + "@ai-sdk/provider": "4.0.14", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", @@ -3473,12 +3473,12 @@ } }, "node_modules/@types/node": { - "version": "26.4.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz", - "integrity": "sha1-PY3IBRWJSVhEjuJmz11rw+UgW9U=", + "version": "26.5.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.1.tgz", + "integrity": "sha512-CzNm2FezW4VR/LjG6yUdiEgLE/rAQ9Slj5gCu/C2VrdcW7I0ahNZ8DRbHT7zOZ6r3ONgd/bsQIeSaoDGrd1C6g==", "license": "MIT", "dependencies": { - "undici-types": "~8.3.0" + "undici-types": "~8.9.0" } }, "node_modules/@types/pg": { @@ -3560,7 +3560,7 @@ "node_modules/@vercel/oidc": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", - "integrity": "sha1-V4Kk1JBEQ/AVgIcFtVN8+cO2hSg=", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", "dev": true, "license": "Apache-2.0", "engines": { @@ -3714,7 +3714,7 @@ "node_modules/@workflow/serde": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", - "integrity": "sha1-gtu68jcPe4rkbZqaSrCRclj422I=", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", "dev": true, "license": "Apache-2.0" }, @@ -3741,15 +3741,15 @@ } }, "node_modules/ai": { - "version": "7.0.93", - "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.93.tgz", - "integrity": "sha1-dEwGMUFcd3LijkQlEyojY/ogk4Q=", + "version": "7.0.99", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.99.tgz", + "integrity": "sha512-Ov+3j/nSajaVH5hO8C94wN9wisG5tAJ8HWsLV9d/+nlzrRGUh3oYO3Kp1fRyUNWjAWLSpGLHLPaHCdfxb307Vg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@ai-sdk/gateway": "4.0.75", - "@ai-sdk/provider": "4.0.10", - "@ai-sdk/provider-utils": "5.0.36" + "@ai-sdk/gateway": "4.0.80", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" }, "engines": { "node": ">=22" @@ -5314,7 +5314,7 @@ "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", - "integrity": "sha1-995M9u+rg4666zI2R0y7paGTCrU=", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", "dev": true, "license": "(AFL-2.1 OR BSD-3-Clause)" }, @@ -6897,7 +6897,7 @@ "node_modules/undici": { "version": "7.29.1", "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", - "integrity": "sha1-d0HG/Is+GkjjAyODNkK/voQUQ60=", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", "dev": true, "license": "MIT", "engines": { @@ -6905,9 +6905,9 @@ } }, "node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha1-ROn8nzJEZIzeo15Pm7LWgelBCAk=", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", "license": "MIT" }, "node_modules/unpipe": { @@ -7418,9 +7418,9 @@ } }, "node_modules/zod": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.5.4.tgz", - "integrity": "sha1-4hXGJCDFKN15UeMftSxUOPH9GEo=", + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 21ac6e6..edf7e15 100644 --- a/package.json +++ b/package.json @@ -57,16 +57,16 @@ "cors": "^2.8.6", "express": "^5.0.1", "winston": "^3.18.3", - "zod": "^4.5.4" + "zod": "^4.6.2" }, "devDependencies": { - "@ai-sdk/azure": "^4.0.63", + "@ai-sdk/azure": "^4.0.68", "@azure/identity": "^4.13.2", "@types/cors": "^2.8.17", "@types/express": "^5.0.0", - "@types/node": "^26.4.1", + "@types/node": "^26.5.1", "@vitest/coverage-v8": "^4.1.11", - "ai": "^7.0.93", + "ai": "^7.0.97", "dotenv": "^17.4.2", "typescript": "^5.9.3", "vitest": "^4.1.11" From 32fad5dce9616f81d00a6e218d60bccb8c921d16 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Mon, 14 Sep 2026 12:54:50 -0700 Subject: [PATCH 18/19] ci(semantic): weekly index refresh -> rolling Release asset Adds .github/workflows/semantic-index.yml: weekly (cron 0 6 * * 1) + manual rebuild of the reviewed-tier index, published as the rolling 'semantic-index-latest' Release asset (~100 MB/week). Release (not LFS-to-main) because main is protected; keeps the blob off clones. Docs updated with consumer steps. --- .github/workflows/semantic-index.yml | 95 ++++++++++++++++++++++++++++ docs/semantic-index-distribution.md | 17 +++++ 2 files changed, 112 insertions(+) create mode 100644 .github/workflows/semantic-index.yml diff --git a/.github/workflows/semantic-index.yml b/.github/workflows/semantic-index.yml new file mode 100644 index 0000000..3c2ca69 --- /dev/null +++ b/.github/workflows/semantic-index.yml @@ -0,0 +1,95 @@ +name: Semantic Index (weekly refresh) + +# Rebuilds the local semantic-search index weekly and publishes it as a rolling +# Release asset (`semantic-index-latest`). Consumers download + extract it instead of +# re-embedding the corpus. A Release asset is used (not an LFS commit to `main`) +# because `main` is protected (signed commits + required PR), so a scheduled job +# cannot push to it — and it keeps the ~96 MB blob off every clone. + +on: + workflow_dispatch: + inputs: + limit: + description: "Advisories to embed (0 = all reviewed ~35k; use a small number for a test)" + required: false + default: "0" + schedule: + - cron: "0 6 * * 1" # Mondays 06:00 UTC — schedule only fires from the default branch + +permissions: + contents: write # create/update the rolling Release asset + +concurrency: + group: semantic-index + cancel-in-progress: false + +jobs: + build-index: + runs-on: ubuntu-latest + timeout-minutes: 60 # a full ~35k rebuild fits comfortably (~30 min budget) + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 22.x + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Build TypeScript + run: npm run build + + - name: Clone advisory database (shallow) + run: | + git clone --depth=1 --branch=main \ + https://github.com/github/advisory-database.git external/advisory-database + + # Cache the ONNX model weights between runs (miss -> downloaded from HF). + - name: Cache embedding model + uses: actions/cache@v4 + with: + path: ~/.cache/semantic-model + key: semantic-model-minilm-v1 + + - name: Build semantic index + env: + ADVISORY_REPO_PATH: ./external/advisory-database + SEMANTIC_MODEL_CACHE: ~/.cache/semantic-model + SEMANTIC_ALLOW_REMOTE: "true" # CI runner may fetch the model (no corp block) + run: | + LIMIT="${{ github.event.inputs.limit }}" + LIMIT="${LIMIT:-0}" + if [ "$LIMIT" = "0" ]; then + node dist/semantic/build-index.js + else + node dist/semantic/build-index.js --limit "$LIMIT" + fi + + - name: Package index + run: | + echo "---- meta.json ----"; cat .semantic-index/meta.json + tar -czf semantic-index.tar.gz -C .semantic-index . + ls -lh semantic-index.tar.gz + + - name: Publish rolling Release asset + env: + GH_TOKEN: ${{ github.token }} + run: | + DBC=$(node -e "process.stdout.write(String((require('./.semantic-index/meta.json').dbCommit)||'unknown'))") + NOTES="Prebuilt local semantic-search index (embeddings.bin + bm25.json + docs.json + meta.json). Rebuilt weekly. advisory-database @ ${DBC}. Extract into .semantic-index/." + if gh release view semantic-index-latest >/dev/null 2>&1; then + gh release upload semantic-index-latest semantic-index.tar.gz --clobber + gh release edit semantic-index-latest --notes "$NOTES" + else + gh release create semantic-index-latest semantic-index.tar.gz \ + --title "Semantic index (latest)" --notes "$NOTES" + fi + + - name: Upload workflow artifact (backup) + uses: actions/upload-artifact@v4 + with: + name: semantic-index + path: .semantic-index/ + retention-days: 14 diff --git a/docs/semantic-index-distribution.md b/docs/semantic-index-distribution.md index 91a2b59..a5593c7 100644 --- a/docs/semantic-index-distribution.md +++ b/docs/semantic-index-distribution.md @@ -3,6 +3,23 @@ Status: prototype (`proto/semantic-search`). Applies to the local hybrid search index in `src/semantic/` (see its README). +## Implemented + +`.github/workflows/semantic-index.yml` runs **weekly** (`cron: "0 6 * * 1"`, plus +manual `workflow_dispatch`): it rebuilds the reviewed-tier index and publishes it as a +rolling GitHub **Release asset** `semantic-index-latest` (`semantic-index.tar.gz`). +A Release asset is used rather than an LFS commit to `main` because `main` is protected +(signed commits + required PR), so a scheduled job cannot push to it — and it keeps the +~96 MB blob off every clone. To consume: + +```bash +gh release download semantic-index-latest -p semantic-index.tar.gz +mkdir -p .semantic-index && tar -xzf semantic-index.tar.gz -C .semantic-index +``` + +The git-lfs channel below remains a valid alternative if you prefer the index to live +in-tree (on a dedicated, unprotected branch). + ## Why redistribute at all Building the reviewed-tier index (~35k advisories) is a one-off CPU cost From 6079494dc803d7b6646b71dffd506814dadc97d8 Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Mon, 14 Sep 2026 13:03:14 -0700 Subject: [PATCH 19/19] ci(semantic): manual maintainer-gated index build -> artifact (no schedule/release yet) workflow_dispatch only; authorize job restricts to admin/maintain; uploads .semantic-index as a downloadable artifact (contents: read, no repo write, no Release). Weekly schedule + distribution deferred and documented. Addresses: run on demand, download as artifact without releasing, and maintainer-only trigger. --- .github/workflows/semantic-index.yml | 60 ++++++++++++++-------------- docs/semantic-index-distribution.md | 50 ++++++++++++++++++----- 2 files changed, 69 insertions(+), 41 deletions(-) diff --git a/.github/workflows/semantic-index.yml b/.github/workflows/semantic-index.yml index 3c2ca69..c1365e3 100644 --- a/.github/workflows/semantic-index.yml +++ b/.github/workflows/semantic-index.yml @@ -1,30 +1,45 @@ -name: Semantic Index (weekly refresh) +name: Semantic Index (manual build) -# Rebuilds the local semantic-search index weekly and publishes it as a rolling -# Release asset (`semantic-index-latest`). Consumers download + extract it instead of -# re-embedding the corpus. A Release asset is used (not an LFS commit to `main`) -# because `main` is protected (signed commits + required PR), so a scheduled job -# cannot push to it — and it keeps the ~96 MB blob off every clone. +# Manually-triggered rebuild of the local semantic-search index. Produces a +# downloadable workflow ARTIFACT (`semantic-index`) — no Release, no repo write. +# Restricted to maintainers/admins via the `authorize` gate below. The weekly +# `schedule` is intentionally NOT enabled yet — see +# docs/semantic-index-distribution.md for how to turn it on and lock it down. on: workflow_dispatch: inputs: limit: - description: "Advisories to embed (0 = all reviewed ~35k; use a small number for a test)" + description: "Advisories to embed (0 = all reviewed ~35k; e.g. 2000 for a quick test)" required: false - default: "0" - schedule: - - cron: "0 6 * * 1" # Mondays 06:00 UTC — schedule only fires from the default branch + default: "2000" permissions: - contents: write # create/update the rolling Release asset + contents: read # read repo + upload artifact only; never writes to the repo concurrency: group: semantic-index cancel-in-progress: false jobs: + # Only maintainers/admins may proceed. workflow_dispatch already requires write + # access; this narrows it to maintain/admin. + authorize: + runs-on: ubuntu-latest + steps: + - name: Require maintainer or admin + env: + GH_TOKEN: ${{ github.token }} + run: | + PERM=$(gh api "repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission" --jq '.permission' 2>/dev/null || echo "unknown") + echo "Actor '${{ github.actor }}' permission: $PERM" + case "$PERM" in + admin|maintain) echo "Authorized." ;; + *) echo "::error::Only maintainers/admins may run this workflow (actor has '$PERM')."; exit 1 ;; + esac + build-index: + needs: authorize runs-on: ubuntu-latest timeout-minutes: 60 # a full ~35k rebuild fits comfortably (~30 min budget) steps: @@ -60,34 +75,19 @@ jobs: SEMANTIC_ALLOW_REMOTE: "true" # CI runner may fetch the model (no corp block) run: | LIMIT="${{ github.event.inputs.limit }}" - LIMIT="${LIMIT:-0}" + LIMIT="${LIMIT:-2000}" if [ "$LIMIT" = "0" ]; then node dist/semantic/build-index.js else node dist/semantic/build-index.js --limit "$LIMIT" fi - - name: Package index + - name: Index stats run: | + ls -lh .semantic-index echo "---- meta.json ----"; cat .semantic-index/meta.json - tar -czf semantic-index.tar.gz -C .semantic-index . - ls -lh semantic-index.tar.gz - - - name: Publish rolling Release asset - env: - GH_TOKEN: ${{ github.token }} - run: | - DBC=$(node -e "process.stdout.write(String((require('./.semantic-index/meta.json').dbCommit)||'unknown'))") - NOTES="Prebuilt local semantic-search index (embeddings.bin + bm25.json + docs.json + meta.json). Rebuilt weekly. advisory-database @ ${DBC}. Extract into .semantic-index/." - if gh release view semantic-index-latest >/dev/null 2>&1; then - gh release upload semantic-index-latest semantic-index.tar.gz --clobber - gh release edit semantic-index-latest --notes "$NOTES" - else - gh release create semantic-index-latest semantic-index.tar.gz \ - --title "Semantic index (latest)" --notes "$NOTES" - fi - - name: Upload workflow artifact (backup) + - name: Upload index artifact uses: actions/upload-artifact@v4 with: name: semantic-index diff --git a/docs/semantic-index-distribution.md b/docs/semantic-index-distribution.md index a5593c7..276266a 100644 --- a/docs/semantic-index-distribution.md +++ b/docs/semantic-index-distribution.md @@ -3,22 +3,50 @@ Status: prototype (`proto/semantic-search`). Applies to the local hybrid search index in `src/semantic/` (see its README). -## Implemented +## Current workflow: manual build → downloadable artifact -`.github/workflows/semantic-index.yml` runs **weekly** (`cron: "0 6 * * 1"`, plus -manual `workflow_dispatch`): it rebuilds the reviewed-tier index and publishes it as a -rolling GitHub **Release asset** `semantic-index-latest` (`semantic-index.tar.gz`). -A Release asset is used rather than an LFS commit to `main` because `main` is protected -(signed commits + required PR), so a scheduled job cannot push to it — and it keeps the -~96 MB blob off every clone. To consume: +`.github/workflows/semantic-index.yml` is a **manually-triggered** build +(`workflow_dispatch`) that rebuilds the index and uploads it as a workflow +**artifact** named `semantic-index` (retention 14 days). It does **not** write to the +repo and does **not** create a Release — so `permissions: contents: read` only, no +branch-protection interaction, no clone bloat. The weekly `schedule` is **deferred** +(not enabled yet). + +Run it: **Actions → “Semantic Index (manual build)” → Run workflow**, pick the branch, +optionally set `limit` (default `2000` for a quick test; `0` = full reviewed ~35k). +Download from the run's **Artifacts**, then: ```bash -gh release download semantic-index-latest -p semantic-index.tar.gz -mkdir -p .semantic-index && tar -xzf semantic-index.tar.gz -C .semantic-index +# unzip the downloaded 'semantic-index' artifact into the repo, then: +mkdir -p .semantic-index && cp -f semantic-index/* .semantic-index/ +``` + +### Who can trigger it (access control) + +- **Baseline:** `workflow_dispatch` can only be started by users with **write** access + — GitHub rejects a dispatch from anyone without it. +- **Narrowed to maintainers/admins:** the `authorize` job checks the actor's + collaborator permission and fails unless it is `admin` or `maintain`. +- **Hard enforcement (recommended, admin-configured):** create a repo **Environment** + named `semantic-index` with **required reviewers** = the maintainer team, then add + `environment: semantic-index` to the `build-index` job. Each run then pauses for + maintainer approval before doing any work. Left un-wired by default so we don't + auto-create an environment. + +### Enabling the weekly schedule later (deferred) + +Add back to the workflow (the `schedule` trigger only fires from the default branch): + +```yaml +on: + schedule: + - cron: "0 6 * * 1" # Mondays 06:00 UTC ``` -The git-lfs channel below remains a valid alternative if you prefer the index to live -in-tree (on a dedicated, unprotected branch). +Scheduled runs execute as the repo (no interactive actor), so guard the `authorize` +actor-check with `if: github.event_name == 'workflow_dispatch'`. For *distributing* a +scheduled build's output, choose a Release asset or a git-lfs branch (below) — **not +wired yet.** ## Why redistribute at all