From 88494e2ebaf2fbaa301da77f22fb6ce00c5f3dc4 Mon Sep 17 00:00:00 2001 From: AmintaCCCP Date: Wed, 29 Jul 2026 21:16:54 +0800 Subject: [PATCH 1/7] feat: add repo license recognition, filtering & MCP search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #251 — recognize repo license (SPDX id) across the stack: GitHub API → DB → AI/embedding/search → UI filter → MCP server. - githubApi.ts / repositories.ts / sync.ts: normalize GitHub license object/string/null to SPDX id string via toLicenseSpdxId - db/schema.ts: add `license` column to repositories - types/index.ts: Repository.license + SearchFilters.licenses - licenseFilter.ts: normalizeLicense (case-insensitive NOASSERTION collapse) + NO_LICENSE_SENTINEL for the "no license" facet - repoSearch.ts / SearchBar.tsx / DataManagementPanel.tsx: license filter facet + basic text search includes license - RepositoryCard.tsx: render Scale badge with SPDX id - vectorSearchService.ts: embed License line + bump format v3 - aiService.ts: include license in AI context & local fallbacks - mcp (server + electron): licenses filter param, byLicense stats, projected license field; normalizeLicense mirrored server-side Co-Authored-By: Claude Haiku 4.5 --- electron/mcpLocalServer.js | 18 ++++++ server/src/db/schema.ts | 1 + server/src/mcp/provider.ts | 6 ++ server/src/mcp/repoSearch.ts | 21 +++++++ server/src/mcp/tools.ts | 5 ++ server/src/routes/repositories.ts | 26 ++++++-- server/src/routes/sync.ts | 17 ++++- src/components/RepositoryCard.tsx | 15 ++++- src/components/SearchBar.test.tsx | 1 + src/components/SearchBar.tsx | 47 ++++++++++++++ .../settings/DataManagementPanel.tsx | 2 + src/services/aiService.ts | 13 ++-- src/services/githubApi.ts | 22 ++++++- src/services/vectorSearchService.ts | 7 ++- src/store/useAppStore.ts | 1 + src/types/index.ts | 4 ++ src/utils/licenseFilter.test.ts | 63 +++++++++++++++++++ src/utils/licenseFilter.ts | 35 +++++++++++ src/utils/repoSearch.test.ts | 16 +++++ src/utils/repoSearch.ts | 10 +++ 20 files changed, 316 insertions(+), 14 deletions(-) create mode 100644 src/utils/licenseFilter.test.ts create mode 100644 src/utils/licenseFilter.ts diff --git a/electron/mcpLocalServer.js b/electron/mcpLocalServer.js index afed44201..1a86ebc89 100644 --- a/electron/mcpLocalServer.js +++ b/electron/mcpLocalServer.js @@ -23,6 +23,7 @@ function performBasicTextSearch(repos, query) { ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', + repo.license || '', ] .join(' ') .toLowerCase(); @@ -30,6 +31,14 @@ function performBasicTextSearch(repos, query) { }); } +// License 归一化镜像（与 src/utils/licenseFilter.ts 一致） +const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; +const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); +function normalizeLicense(v) { + if (!v) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; +} + function projectRepo(repo, max = 400) { const summary = repo.ai_summary || repo.custom_description || repo.description || null; const truncated = @@ -54,6 +63,7 @@ function projectRepo(repo, max = 400) { starred_at: repo.starred_at, updated_at: repo.updated_at, pushed_at: repo.pushed_at, + license: repo.license ?? null, }; } @@ -265,6 +275,11 @@ function getTools(vectorAvailable) { query: { type: 'string' }, languages: { type: 'array', items: { type: 'string' } }, tags: { type: 'array', items: { type: 'string' } }, + licenses: { + type: 'array', + items: { type: 'string' }, + description: 'SPDX id list (e.g. ["MIT","Apache-2.0"]); use "__NO_LICENSE__" for repos with no license', + }, category: { type: 'string' }, minStars: { type: 'number' }, maxStars: { type: 'number' }, @@ -362,6 +377,9 @@ async function callTool(name, args, snapshot) { return args.tags.some((t) => tags.includes(t)); }); } + if (args?.licenses?.length) { + list = list.filter((r) => args.licenses.includes(normalizeLicense(r.license))); + } if (args?.category) { list = list.filter((r) => r.custom_category === args.category); } diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index e45b59311..2ebfc3215 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -150,4 +150,5 @@ export function initializeSchema(db: Database.Database): void { addColumnIfMissing(db, 'vector_search_configs', 'index_mode', "TEXT NOT NULL DEFAULT 'readme'"); addColumnIfMissing(db, 'vector_search_configs', 'readme_max_chars', 'INTEGER NOT NULL DEFAULT 6000'); addColumnIfMissing(db, 'repositories', 'vector_indexed_at', 'TEXT'); + addColumnIfMissing(db, 'repositories', 'license', 'TEXT'); } diff --git a/server/src/mcp/provider.ts b/server/src/mcp/provider.ts index e8e7924c5..3a9863789 100644 --- a/server/src/mcp/provider.ts +++ b/server/src/mcp/provider.ts @@ -5,6 +5,7 @@ import { logger } from '../services/logger.js'; import { type McpRepository, type McpSearchFilters, + normalizeLicense, projectRepoForAgent, searchRepositories, } from './repoSearch.js'; @@ -47,6 +48,7 @@ export function transformRepoRow(row: Record): McpRepository { custom_category: (row.custom_category as string | null) ?? null, category_locked: !!row.category_locked, subscribed_to_releases: !!row.subscribed_to_releases, + license: (row.license as string | null) ?? null, }; } @@ -107,6 +109,7 @@ export function searchRepos(filters: McpSearchFilters) { export function getStats() { const repos = loadAllRepositories(); const byLanguage: Record = {}; + const byLicense: Record = {}; const tagCounts: Record = {}; let analyzed = 0; let subscribed = 0; @@ -115,6 +118,8 @@ export function getStats() { for (const r of repos) { const lang = r.language || 'Unknown'; byLanguage[lang] = (byLanguage[lang] || 0) + 1; + const lic = normalizeLicense(r.license); + byLicense[lic] = (byLicense[lic] || 0) + 1; if (r.analyzed_at && !r.analysis_failed) analyzed += 1; if (r.analyzed_at && r.analysis_failed) failed += 1; if (r.subscribed_to_releases) subscribed += 1; @@ -135,6 +140,7 @@ export function getStats() { unanalyzed: repos.length - analyzed - failed, subscribedToReleases: subscribed, byLanguage, + byLicense, topTags, }; } diff --git a/server/src/mcp/repoSearch.ts b/server/src/mcp/repoSearch.ts index ddc6a5ea8..b307f43dd 100644 --- a/server/src/mcp/repoSearch.ts +++ b/server/src/mcp/repoSearch.ts @@ -3,6 +3,17 @@ * Kept server-local to avoid coupling the Express package to the Vite app tree. */ +/** + * License 归一化的服务端镜像（与 src/utils/licenseFilter.ts 保持一致）。 + * 服务端 MCP 无法 import src/ 树，故此处保留一份相同实现；改前端那份时请一并同步。 + */ +export const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; +const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); +export function normalizeLicense(v: string | null | undefined): string { + if (!v) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; +} + export interface McpRepository { id: number; name: string; @@ -27,6 +38,7 @@ export interface McpRepository { category_locked?: boolean; subscribed_to_releases?: boolean; owner?: { login: string; avatar_url?: string }; + license?: string | null; } export interface McpSearchFilters { @@ -43,6 +55,8 @@ export interface McpSearchFilters { isCategoryLocked?: boolean; analysisFailed?: boolean; category?: string; + /** SPDX id 过滤；含 {@link NO_LICENSE_SENTINEL} 表示「无/未声明 license」。 */ + licenses?: string[]; limit?: number; offset?: number; } @@ -65,6 +79,7 @@ export function performBasicTextSearch(repos: T[], quer ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', + repo.license || '', ] .join(' ') .toLowerCase(); @@ -108,6 +123,11 @@ export function applyRepoFilters( return filters.platforms!.some((p) => platforms.includes(p)); }); } + if (filters.licenses?.length) { + filtered = filtered.filter((r) => + filters.licenses!.includes(normalizeLicense(r.license)) + ); + } if (filters.isAnalyzed !== undefined && filters.analysisFailed === undefined) { filtered = filtered.filter((r) => filters.isAnalyzed ? !!r.analyzed_at && !r.analysis_failed : !r.analyzed_at @@ -196,5 +216,6 @@ export function projectRepoForAgent( starred_at: repo.starred_at, updated_at: repo.updated_at, pushed_at: repo.pushed_at, + license: repo.license ?? null, }; } diff --git a/server/src/mcp/tools.ts b/server/src/mcp/tools.ts index 7b334c9d7..850c41290 100644 --- a/server/src/mcp/tools.ts +++ b/server/src/mcp/tools.ts @@ -54,6 +54,10 @@ export function registerMcpTools(server: McpServer): void { languages: z.array(z.string()).optional(), tags: z.array(z.string()).optional(), platforms: z.array(z.string()).optional(), + licenses: z + .array(z.string()) + .optional() + .describe('SPDX id list (e.g. ["MIT","Apache-2.0"]); use "__NO_LICENSE__" for repos with no license'), category: z.string().optional().describe('custom_category exact match'), minStars: z.number().optional(), maxStars: z.number().optional(), @@ -71,6 +75,7 @@ export function registerMcpTools(server: McpServer): void { languages: args.languages, tags: args.tags, platforms: args.platforms, + licenses: args.licenses, category: args.category, minStars: args.minStars, maxStars: args.maxStars, diff --git a/server/src/routes/repositories.ts b/server/src/routes/repositories.ts index dbf6779f1..0389ff94c 100644 --- a/server/src/routes/repositories.ts +++ b/server/src/routes/repositories.ts @@ -12,6 +12,21 @@ function parseJsonColumn(value: unknown): unknown[] { } catch { return []; } } +/** + * 把 GitHub 的 license 值统一为 SPDX id 字符串或 null。 + * 接受三种形态：GitHub 原始对象 `{ key, spdx_id, name, url }`、已规范化的字符串、null。 + * 优先取 spdx_id（如 "MIT"），无则回退 key（如 "Other" → "NOASSERTION" 由前端归一化处理）。 + */ +function toLicenseSpdxId(license: unknown): string | null { + if (license == null) return null; + if (typeof license === 'string') return license || null; + if (typeof license === 'object') { + const l = license as { spdx_id?: string; key?: string }; + return l.spdx_id || l.key || null; + } + return null; +} + /** Transform a database row into the API response shape, parsing JSON columns. */ function transformRepo(row: Record) { return { @@ -40,6 +55,7 @@ function transformRepo(row: Record) { last_edited: row.last_edited, subscribed_to_releases: !!row.subscribed_to_releases, vector_indexed_at: row.vector_indexed_at ?? undefined, + license: row.license ?? null, }; } @@ -134,8 +150,8 @@ router.put('/api/repositories', (req, res) => { owner_login, owner_avatar_url, topics, ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, - subscribed_to_releases, vector_indexed_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + subscribed_to_releases, vector_indexed_at, license + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name = excluded.name, full_name = excluded.full_name, @@ -161,7 +177,8 @@ router.put('/api/repositories', (req, res) => { category_locked = excluded.category_locked, last_edited = CASE WHEN excluded.last_edited IS NOT NULL AND excluded.last_edited != '' THEN excluded.last_edited ELSE repositories.last_edited END, subscribed_to_releases = excluded.subscribed_to_releases, - vector_indexed_at = excluded.vector_indexed_at + vector_indexed_at = excluded.vector_indexed_at, + license = excluded.license `); const deleteAllReleases = db.prepare('DELETE FROM releases'); @@ -208,7 +225,8 @@ router.put('/api/repositories', (req, res) => { JSON.stringify(Array.isArray(repo.custom_tags) ? repo.custom_tags : []), repo.custom_category ?? null, (repo.category_locked === true || repo.category_locked === 1) ? 1 : 0, repo.last_edited ?? null, (repo.subscribed_to_releases === true || repo.subscribed_to_releases === 1) ? 1 : 0, - repo.vector_indexed_at ?? null + repo.vector_indexed_at ?? null, + toLicenseSpdxId(repo.license) ); count++; } diff --git a/server/src/routes/sync.ts b/server/src/routes/sync.ts index 265a0fa30..61bfa4939 100644 --- a/server/src/routes/sync.ts +++ b/server/src/routes/sync.ts @@ -108,14 +108,24 @@ router.post('/api/sync/import', (req, res) => { owner_login, owner_avatar_url, topics, ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, - subscribed_to_releases, vector_indexed_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + subscribed_to_releases, vector_indexed_at, license + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); for (const r of repos) { // 验证必需的字段 if (!r.id || typeof r.id !== 'number') { throw new Error(`Invalid repository data: missing or invalid id`); } + // license：兼容旧备份（无该列→null）、GitHub 对象形态、已规范化的 SPDX 字符串 + const rawLicense = (r as Record).license; + const licenseValue = + typeof rawLicense === 'string' + ? rawLicense || null + : rawLicense && typeof rawLicense === 'object' + ? (rawLicense as { spdx_id?: string; key?: string }).spdx_id || + (rawLicense as { key?: string }).key || + null + : null; repoStmt.run( r.id, r.name, r.full_name, r.description ?? null, r.html_url, r.stargazers_count ?? 0, r.language ?? null, @@ -131,7 +141,8 @@ router.post('/api/sync/import', (req, res) => { typeof r.custom_tags === 'string' ? r.custom_tags : JSON.stringify(r.custom_tags ?? []), r.custom_category ?? null, (r.category_locked === true || r.category_locked === 1) ? 1 : 0, r.last_edited ?? null, r.subscribed_to_releases ? 1 : 0, - r.vector_indexed_at ?? null + r.vector_indexed_at ?? null, + licenseValue ); } counts.repositories = repos.length; diff --git a/src/components/RepositoryCard.tsx b/src/components/RepositoryCard.tsx index 8b04c3bab..0bcd3d825 100644 --- a/src/components/RepositoryCard.tsx +++ b/src/components/RepositoryCard.tsx @@ -1,6 +1,6 @@ import React, { useState, useRef, useEffect, useMemo, useCallback } from 'react'; import { createPortal } from 'react-dom'; -import { GripVertical, Star, StarOff, ExternalLink, Calendar, Bell, BellOff, Bot, Sparkles, Monitor, Smartphone, Globe, Terminal, Package, Edit3, BookOpen, Apple, Square, CheckSquare, Loader2, HelpCircle, Search } from 'lucide-react'; +import { GripVertical, Star, StarOff, ExternalLink, Calendar, Bell, BellOff, Bot, Sparkles, Monitor, Smartphone, Globe, Terminal, Package, Edit3, BookOpen, Apple, Square, CheckSquare, Loader2, HelpCircle, Search, Scale } from 'lucide-react'; import { Repository, Category } from '../types'; import { useAppStore } from '../store/useAppStore'; import { EmbeddingClient, VectorSearchService, findSimilarRepositories } from '../services/vectorSearchService'; @@ -12,6 +12,7 @@ import { formatDistanceToNow } from 'date-fns'; import { RepositoryEditModal } from './RepositoryEditModal'; import { ReadmeModal } from './ReadmeModal'; import { FloatingTooltip } from './FloatingTooltip'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; import { shallow } from 'zustand/shallow'; import { useDialog } from '../hooks/useDialog'; import { logger } from '../services/logger'; @@ -1136,6 +1137,17 @@ const RepositoryCardComponent: React.FC = ({ {formatNumber(repository.stargazers_count)} + {(() => { + // license：归一化后展示 SPDX id；无 license 不渲染 + const lic = normalizeLicense(repository.license); + if (lic === NO_LICENSE_SENTINEL) return null; + return ( +
+ + {lic} +
+ ); + })()} {/* Update Time / 查找相似仓库 - 悬停时时间淡出，显示高亮按钮 */} @@ -1232,6 +1244,7 @@ export const RepositoryCard = React.memo(RepositoryCardComponent, (prevProps, ne prevProps.repository.category_locked === nextProps.repository.category_locked && prevProps.repository.description === nextProps.repository.description && prevProps.repository.topics === nextProps.repository.topics && + prevProps.repository.license === nextProps.repository.license && prevProps.repository.stargazers_count === nextProps.repository.stargazers_count && prevProps.repository.pushed_at === nextProps.repository.pushed_at && prevProps.repository.updated_at === nextProps.repository.updated_at && diff --git a/src/components/SearchBar.test.tsx b/src/components/SearchBar.test.tsx index 4bbef5cff..fde9054da 100644 --- a/src/components/SearchBar.test.tsx +++ b/src/components/SearchBar.test.tsx @@ -48,6 +48,7 @@ const defaultSearchFilters: SearchFilters = { tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', }; diff --git a/src/components/SearchBar.tsx b/src/components/SearchBar.tsx index a0b4a417a..f8cb19d74 100644 --- a/src/components/SearchBar.tsx +++ b/src/components/SearchBar.tsx @@ -9,6 +9,7 @@ import { useSearchShortcuts } from '../hooks/useSearchShortcuts'; import { useDialog } from '../hooks/useDialog'; import { isRepoCustomized } from '../utils/repoUtils'; import { applyRepoFilters, performBasicTextSearch as basicTextSearch, sortRepositories } from '../utils/repoSearch'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; import { NumberInput } from './ui/NumberInput'; type SortBy = 'stars' | 'updated' | 'name' | 'starred'; @@ -107,6 +108,7 @@ export const SearchBar: React.FC = () => { const [availableLanguages, setAvailableLanguages] = useState([]); const [availableTags, setAvailableTags] = useState([]); const [availablePlatforms, setAvailablePlatforms] = useState([]); + const [availableLicenses, setAvailableLicenses] = useState([]); const [isRealTimeSearch, setIsRealTimeSearch] = useState(false); const [isComposing, setIsComposing] = useState(false); @@ -186,10 +188,17 @@ export const SearchBar: React.FC = () => { ...repositories.flatMap(r => r.custom_tags || []) ])]; const platforms = [...new Set(repositories.flatMap(r => r.ai_platforms || []))] as string[]; + // 开源许可：归一化为 SPDX id 或 NO_LICENSE_SENTINEL，排序并把「无」项放最后 + const licenses = [...new Set(repositories.map(r => normalizeLicense(r.license)))].sort((a, b) => { + if (a === NO_LICENSE_SENTINEL) return 1; + if (b === NO_LICENSE_SENTINEL) return -1; + return a.localeCompare(b); + }); setAvailableLanguages(languages); setAvailableTags(tags); setAvailablePlatforms(platforms); + setAvailableLicenses(licenses); // Generate search suggestions from available data const suggestions = [ @@ -640,6 +649,14 @@ export const SearchBar: React.FC = () => { setSearchFilters({ platforms: newPlatforms }); }; + const handleLicenseToggle = (license: string) => { + const current = searchFilters.licenses ?? []; + const newLicenses = current.includes(license) + ? current.filter(l => l !== license) + : [...current, license]; + setSearchFilters({ licenses: newLicenses }); + }; + const clearFilters = () => { setSearchQuery(''); setIsRealTimeSearch(false); @@ -648,6 +665,7 @@ export const SearchBar: React.FC = () => { tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', minStars: undefined, @@ -664,6 +682,7 @@ export const SearchBar: React.FC = () => { searchFilters.languages.length + searchFilters.tags.length + searchFilters.platforms.length + + (searchFilters.licenses?.length ?? 0) + (searchFilters.minStars !== undefined ? 1 : 0) + (searchFilters.maxStars !== undefined ? 1 : 0) + (searchFilters.isAnalyzed !== undefined ? 1 : 0) + @@ -748,6 +767,8 @@ export const SearchBar: React.FC = () => { starred_at: newRepo.starred_at, owner: newRepo.owner, topics: newRepo.topics, + // 回填历史仓库缺失的 license 字段（GitHub 源元数据，跟随 newRepo） + license: newRepo.license ?? null, }; } return newRepo; @@ -1236,6 +1257,32 @@ export const SearchBar: React.FC = () => { )} + {/* Licenses */} + {availableLicenses.length > 0 && ( +
+

+ {t('开源许可', 'License')} +

+
+ {availableLicenses.map(license => ( + + ))} +
+
+ )} + {/* Tags */} {availableTags.length > 0 && (
diff --git a/src/components/settings/DataManagementPanel.tsx b/src/components/settings/DataManagementPanel.tsx index 8b922d561..d74780472 100644 --- a/src/components/settings/DataManagementPanel.tsx +++ b/src/components/settings/DataManagementPanel.tsx @@ -442,6 +442,7 @@ export const DataManagementPanel: React.FC = ({ t }) = tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', isAnalyzed: undefined, @@ -968,6 +969,7 @@ export const DataManagementPanel: React.FC = ({ t }) = tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', isAnalyzed: undefined, diff --git a/src/services/aiService.ts b/src/services/aiService.ts index 45f1df293..3918553b8 100644 --- a/src/services/aiService.ts +++ b/src/services/aiService.ts @@ -738,6 +738,7 @@ AI Summary: ${gist.ai_summary || 'None'}`; const parts = [`${index + 1}. ID: ${repo.id} | ${repo.full_name}`]; if (desc) parts.push(` ${desc}`); const meta = [repo.language, `★${stars}`]; + if (repo.license) meta.push(`License: ${repo.license}`); if (tags) meta.push(`Tags: ${tags}`); parts.push(` ${meta.join(' | ')}`); return parts.join('\n'); @@ -1315,7 +1316,8 @@ ${repoInfo} aiTags: (repo.ai_tags || []).join(' ').toLowerCase(), aiPlatforms: (repo.ai_platforms || []).join(' ').toLowerCase(), customDescription: (repo.custom_description || '').toLowerCase(), - customTags: (repo.custom_tags || []).join(' ').toLowerCase() + customTags: (repo.custom_tags || []).join(' ').toLowerCase(), + license: (repo.license || '').toLowerCase(), }; // Check if any query word matches any field @@ -1434,8 +1436,9 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), + repo.license || '', ].join(' ').toLowerCase(); - + // Check if any of the AI-enhanced terms match return allSearchTerms.some(term => { const normalizedTerm = term.toLowerCase(); @@ -1459,8 +1462,9 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), + repo.license || '', ].join(' ').toLowerCase(); - + // Split query into words and check if all words are present const queryWords = normalizedQuery.split(/\s+/); return queryWords.every(word => searchableText.includes(word)); @@ -1483,8 +1487,9 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), + repo.license || '', ].join(' ').toLowerCase(); - + // Split query into words and check if all words are present const queryWords = normalizedQuery.split(/\s+/); return queryWords.every(word => searchableText.includes(word)); diff --git a/src/services/githubApi.ts b/src/services/githubApi.ts index 02c33bd6a..5db30f3c4 100644 --- a/src/services/githubApi.ts +++ b/src/services/githubApi.ts @@ -40,6 +40,21 @@ interface GitHubStarredItem { [key: string]: unknown; } +/** + * 把 GitHub 返回的 license 值统一为 SPDX id 字符串或 null。 + * 接受 GitHub 原始对象 `{ key, spdx_id, name, url }`、已规范化的字符串、null。 + * 优先取 spdx_id（如 "MIT"），无则回退 key（如 "Other"）。 + */ +function toLicenseSpdxId(license: unknown): string | null { + if (license == null) return null; + if (typeof license === 'string') return license || null; + if (typeof license === 'object') { + const l = license as { spdx_id?: string; key?: string }; + return l.spdx_id || l.key || null; + } + return null; +} + interface GitHubRateLimitResponse { rate: { remaining: number; @@ -301,10 +316,15 @@ export class GitHubApiService { if (item.starred_at && item.repo) { return { ...item.repo, + // 归一化 GitHub 的 license 值（对象/字符串/null）为 SPDX id 字符串或 null + license: toLicenseSpdxId(item.repo.license), starred_at: item.starred_at }; } - return item; + return { + ...item, + license: toLicenseSpdxId(item.license), + }; }) as T; } diff --git a/src/services/vectorSearchService.ts b/src/services/vectorSearchService.ts index 8ff6dafa3..8043f2df4 100644 --- a/src/services/vectorSearchService.ts +++ b/src/services/vectorSearchService.ts @@ -187,6 +187,7 @@ export interface VectorizeVector { language: string; stars: number; tags: string[]; + license: string; }; } @@ -199,6 +200,7 @@ export interface VectorQueryResult { language: string; stars: number; tags: string[]; + license: string; }; } @@ -323,7 +325,7 @@ export class VectorSearchService { * buildEmbeddingText 的输出格式变化时必须递增， * 使增量索引能检测到格式变化并强制重新索引所有向量。 */ -export const EMBEDDING_FORMAT_VERSION = 2; +export const EMBEDDING_FORMAT_VERSION = 3; /** * 拼接仓库文本用于 embedding @@ -357,6 +359,8 @@ export function buildEmbeddingText(repo: Repository, readmeContent?: string, max if (allTopics.length > 0) parts.push(`Topics: ${allTopics.join(', ')}`); if (repo.language) parts.push(`Language: ${repo.language}`); + // 开源许可：SPDX id（如 MIT），短文本，对 embedding token 成本可忽略 + if (repo.license) parts.push(`License: ${repo.license}`); // README 内容提供最丰富的语义信息 if (readmeContent) { @@ -585,6 +589,7 @@ export async function indexAllRepos( language: batch[j].language || '', stars: batch[j].stargazers_count || 0, tags: batch[j].ai_tags || [], + license: batch[j].license || '', }, }); } else { diff --git a/src/store/useAppStore.ts b/src/store/useAppStore.ts index 282e8e6c1..dee69ac25 100644 --- a/src/store/useAppStore.ts +++ b/src/store/useAppStore.ts @@ -470,6 +470,7 @@ const initialSearchFilters: SearchFilters = { tags: [], languages: [], platforms: [], + licenses: [], sortBy: 'stars', sortOrder: 'desc', isAnalyzed: undefined, diff --git a/src/types/index.ts b/src/types/index.ts index ca25533d4..fe61a40f0 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -32,6 +32,8 @@ export interface Repository { vector_indexed_at?: string; // ISO timestamp of last successful vector indexing last_release_fetch_time?: string; // ISO timestamp, for incremental sync has_fetched_releases?: boolean; // whether this repo has been synced for releases + /** SPDX id（如 'MIT'、'Apache-2.0'）；无许可证/未识别为 null。AI/搜索/过滤均以此为准。 */ + license?: string | null; } export interface ReleaseAsset { @@ -344,6 +346,8 @@ export interface SearchFilters { isEdited?: boolean; // 新增：是否已编辑 isCategoryLocked?: boolean; // 新增：分类是否已锁定 analysisFailed?: boolean; // 新增：分析是否失败 + /** SPDX id 过滤；过滤面板可采用 `NO_LICENSE_SENTINEL` 表示「无/未声明 license」。 */ + licenses: string[]; // 新增：开源许可过滤 } export interface Category { diff --git a/src/utils/licenseFilter.test.ts b/src/utils/licenseFilter.test.ts new file mode 100644 index 000000000..33c76ee5c --- /dev/null +++ b/src/utils/licenseFilter.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { buildEmbeddingText } from '../services/vectorSearchService'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from './licenseFilter'; +import type { Repository } from '../types'; + +const baseRepo = (overrides: Partial = {}): Repository => ({ + id: 1, + name: 'repo-1', + full_name: 'owner/repo-1', + description: 'A test repo', + html_url: 'https://github.com/owner/repo-1', + stargazers_count: 10, + forks_count: 1, + forks: 1, + language: 'TypeScript', + created_at: '2026-01-01T00:00:00.000Z', + updated_at: '2026-01-02T00:00:00.000Z', + pushed_at: '2026-01-03T00:00:00.000Z', + owner: { login: 'owner', avatar_url: 'https://github.com/a.png' }, + topics: ['test'], + ...overrides, +}); + +describe('normalizeLicense', () => { + it('passes through SPDX ids unchanged', () => { + expect(normalizeLicense('MIT')).toBe('MIT'); + expect(normalizeLicense('Apache-2.0')).toBe('Apache-2.0'); + expect(normalizeLicense('GPL-3.0')).toBe('GPL-3.0'); + }); + + it('collapses null/undefined/empty to the no-license sentinel', () => { + expect(normalizeLicense(null)).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(undefined)).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('')).toBe(NO_LICENSE_SENTINEL); + }); + + it('collapses GitHub "no assertion" forms to the sentinel', () => { + expect(normalizeLicense('NOASSERTION')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('Other')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('NONE')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('no-license')).toBe(NO_LICENSE_SENTINEL); + }); + + it('collapses lowercase variants (legacy backup / third-party sources)', () => { + expect(normalizeLicense('noassertion')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('other')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('none')).toBe(NO_LICENSE_SENTINEL); + }); +}); + +describe('buildEmbeddingText license', () => { + it('includes a License: line when license is set', () => { + const text = buildEmbeddingText(baseRepo({ license: 'MIT' })); + expect(text).toContain('License: MIT'); + }); + + it('omits the License line when license is null/missing', () => { + const textNull = buildEmbeddingText(baseRepo({ license: null })); + const textMissing = buildEmbeddingText(baseRepo()); + expect(textNull).not.toContain('License:'); + expect(textMissing).not.toContain('License:'); + }); +}); diff --git a/src/utils/licenseFilter.ts b/src/utils/licenseFilter.ts new file mode 100644 index 000000000..c3c103fce --- /dev/null +++ b/src/utils/licenseFilter.ts @@ -0,0 +1,35 @@ +/** + * License 过滤与归一化工具。 + * + * 仓库元数据中 license 存为 SPDX id 字符串（如 "MIT"、"Apache-2.0"）或 null。 + * GitHub 对未声明 / 无法识别许可证的仓库返回 `{ key: 'Other', spdx_id: 'NOASSERTION' }`， + * 我们存 spdx_id 即 `'NOASSERTION'`。为让过滤面板提供一个「无/未声明」聚合项， + * 统一把这些情形归一化为 {@link NO_LICENSE_SENTINEL}。 + * + * 本模块同时被前端 UI、过滤求值、MCP 等多处复用；服务端 MCP 因无法 import src/ 树， + * 在 server/src/mcp/repoSearch.ts 内保留一份相同实现，改这里时请一并同步。 + */ + +/** 「无 license」聚合哨兵：用于过滤器把 null / NOASSERTION / Other 等归并为一项。 */ +export const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; + +/** + * 视作「无/未声明 license」的值集合（大小写不敏感比对，覆盖常见 GitHub/SPDX 写法）。 + * - `''` 空串 + * - `'noassertion'` GitHub「无 SPDX 断言」的 spdx_id（NOASSERTION） + * - `'other'` GitHub license.key（Other，无 SPDX 时） + * - `'none'` SPDX「无 license」（NONE） + * - `'no-license'` 兜底串 + */ +const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); + +/** + * 把仓库的 license 值归一化为「SPDX id」或「无 license 哨兵」。 + * 比对大小写不敏感，以收敛历史备份/第三方源写入的小写变体（如 'other'、'none'）。 + * @param v 原始 license 值（SPDX id / 哨兵值 / null / undefined / 空串） + * @returns 归一化后的字符串；无 license 时返回 {@link NO_LICENSE_SENTINEL} + */ +export function normalizeLicense(v: string | null | undefined): string { + if (!v) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; +} diff --git a/src/utils/repoSearch.test.ts b/src/utils/repoSearch.test.ts index 841696f3d..7ef1489a6 100644 --- a/src/utils/repoSearch.test.ts +++ b/src/utils/repoSearch.test.ts @@ -6,6 +6,7 @@ import { searchRepositories, projectRepoForAgent, } from './repoSearch'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from './licenseFilter'; function makeRepo(partial: Partial & Pick): Repository { return { @@ -36,6 +37,7 @@ const sample: Repository[] = [ ai_tags: ['crdt', 'sync'], ai_platforms: ['cli'], topics: ['database'], + license: 'MIT', }), makeRepo({ id: 2, @@ -46,6 +48,7 @@ const sample: Repository[] = [ stargazers_count: 50, ai_tags: ['webdav'], custom_category: 'tools', + license: 'GPL-3.0', }), makeRepo({ id: 3, @@ -56,6 +59,7 @@ const sample: Repository[] = [ stargazers_count: 200, analyzed_at: '2024-01-01', analysis_failed: true, + license: 'NOASSERTION', }), ]; @@ -90,6 +94,18 @@ describe('applyRepoFilters', () => { const hits = applyRepoFilters(sample, { sortBy: 'stars', sortOrder: 'desc' }); expect(hits.map((r) => r.id)).toEqual([1, 3, 2]); }); + + it('filters by SPDX id license', () => { + const hits = applyRepoFilters(sample, { licenses: ['MIT'] }); + expect(hits.map((r) => r.id)).toEqual([1]); + }); + + it('aggregates no-license repos via the sentinel', () => { + // normalizeLicense collapses 'NOASSERTION' (and null/'Other') to the sentinel + expect(normalizeLicense('NOASSERTION')).toBe(NO_LICENSE_SENTINEL); + const hits = applyRepoFilters(sample, { licenses: [NO_LICENSE_SENTINEL] }); + expect(hits.map((r) => r.id)).toEqual([3]); + }); }); describe('searchRepositories', () => { diff --git a/src/utils/repoSearch.ts b/src/utils/repoSearch.ts index 64eeafb64..a59fd4b80 100644 --- a/src/utils/repoSearch.ts +++ b/src/utils/repoSearch.ts @@ -1,5 +1,6 @@ import type { Category, Repository, SearchFilters } from '../types'; import { isRepoCustomized } from './repoUtils'; +import { normalizeLicense } from './licenseFilter'; /** Partial filters used by MCP and UI search (all fields optional except when provided). */ export type RepoSearchFilterInput = Partial & { @@ -27,6 +28,7 @@ export function performBasicTextSearch(repos: T[], query: ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', + repo.license || '', ] .join(' ') .toLowerCase(); @@ -117,6 +119,14 @@ export function applyRepoFilters( }); } + const licenses = searchFilters.licenses ?? []; + if (licenses.length > 0) { + // 归一化后比对：SPDX id 精确匹配，无 license（含 NOASSERTION/Other/null）落入 NO_LICENSE_SENTINEL + filtered = filtered.filter((repo) => + licenses.includes(normalizeLicense(repo.license)) + ); + } + if (searchFilters.isAnalyzed !== undefined && searchFilters.analysisFailed === undefined) { filtered = filtered.filter((repo) => searchFilters.isAnalyzed From 965c01ecac966cc50f05c11a9ece2b0088b0ab07 Mon Sep 17 00:00:00 2001 From: AmintaCCCP Date: Wed, 29 Jul 2026 22:35:39 +0800 Subject: [PATCH 2/7] fix: address CodeRabbit audit findings for license feature (PR #254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1 Electron-local MCP: track byLicense in gsm_stats, normalize via sentinel #2 repositories.ts UPSERT: preserve stored license when payload omits the field (旧客户端/旧备份), only overwrite when explicitly provided (incl. null) #3 sync.ts import: validate spdx_id/key are strings at runtime before bind, so malformed object values can't poison the import transaction #4 SearchBar: add searchFilters.licenses to the search effect deps array #5 aiService enhanced basic search: score license matches (+0.2) so a license-matching repo outranks a higher-star non-match on mixed queries #6 githubApi: normalize license (toLicenseSpdxId) for watched-repo returns, consistent with /user/starred path #7 Text-search corpora (repoSearch ×3: src/ + server MCP + electron local): use normalizeLicense instead of raw repo.license #8 Vector incremental invalidation: add vector_indexed_license fingerprint column (schema + Repository type); skip predicate + UI mirrors reindex when license changes (normalizeLicense sentinel-stable); indexers stamp it alongside vector_indexed_at; sync/preserve semantics on bulk upsert #9 Expand no-license sentinel tests to cover null/'Other'/'none'/'NOASSERTION' and mixed-case 'noassertion' Tests: vitest 134 passed (adds aiService license-ranking regression + 3 vector license-invalidation regressions). tsc clean for frontend and server. Co-Authored-By: Claude Opus 4.8 --- electron/mcpLocalServer.js | 6 +- server/src/db/schema.ts | 3 + server/src/mcp/repoSearch.ts | 2 +- server/src/routes/repositories.ts | 23 ++++- server/src/routes/sync.ts | 29 +++--- src/components/SearchBar.tsx | 2 +- .../settings/VectorSearchSettings.tsx | 39 ++++++-- src/services/aiService.test.ts | 67 ++++++++++++++ src/services/aiService.ts | 3 + src/services/githubApi.ts | 5 +- src/services/vectorSearchService.test.ts | 88 +++++++++++++++++++ src/services/vectorSearchService.ts | 7 +- src/types/index.ts | 2 + src/utils/repoSearch.test.ts | 14 ++- src/utils/repoSearch.ts | 2 +- 15 files changed, 263 insertions(+), 29 deletions(-) create mode 100644 src/services/aiService.test.ts diff --git a/electron/mcpLocalServer.js b/electron/mcpLocalServer.js index 1a86ebc89..2bc37c713 100644 --- a/electron/mcpLocalServer.js +++ b/electron/mcpLocalServer.js @@ -23,7 +23,7 @@ function performBasicTextSearch(repos, query) { ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', - repo.license || '', + normalizeLicense(repo.license), ] .join(' ') .toLowerCase(); @@ -418,11 +418,14 @@ async function callTool(name, args, snapshot) { } case 'gsm_stats': { const byLanguage = {}; + const byLicense = {}; let analyzed = 0; let subscribed = 0; for (const r of repos) { const lang = r.language || 'Unknown'; byLanguage[lang] = (byLanguage[lang] || 0) + 1; + const lic = normalizeLicense(r.license); + byLicense[lic] = (byLicense[lic] || 0) + 1; if (r.analyzed_at && !r.analysis_failed) analyzed += 1; if (r.subscribed_to_releases) subscribed += 1; } @@ -431,6 +434,7 @@ async function callTool(name, args, snapshot) { analyzed, subscribedToReleases: subscribed, byLanguage, + byLicense, }); } case 'gsm_vector_search': { diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index 2ebfc3215..caa02568e 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -151,4 +151,7 @@ export function initializeSchema(db: Database.Database): void { addColumnIfMissing(db, 'vector_search_configs', 'readme_max_chars', 'INTEGER NOT NULL DEFAULT 6000'); addColumnIfMissing(db, 'repositories', 'vector_indexed_at', 'TEXT'); addColumnIfMissing(db, 'repositories', 'license', 'TEXT'); + // 上一次向量索引时采用的 license 值（SPDX id / null）。用于增量谓词判断 license 是否 + // 变化：当期 license 与此值不一致时需重新索引，保证 license 变更能使向量元数据失效。 + addColumnIfMissing(db, 'repositories', 'vector_indexed_license', 'TEXT'); } diff --git a/server/src/mcp/repoSearch.ts b/server/src/mcp/repoSearch.ts index b307f43dd..8aa5679cd 100644 --- a/server/src/mcp/repoSearch.ts +++ b/server/src/mcp/repoSearch.ts @@ -79,7 +79,7 @@ export function performBasicTextSearch(repos: T[], quer ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', - repo.license || '', + normalizeLicense(repo.license), ] .join(' ') .toLowerCase(); diff --git a/server/src/routes/repositories.ts b/server/src/routes/repositories.ts index 0389ff94c..06e7b083b 100644 --- a/server/src/routes/repositories.ts +++ b/server/src/routes/repositories.ts @@ -56,6 +56,7 @@ function transformRepo(row: Record) { subscribed_to_releases: !!row.subscribed_to_releases, vector_indexed_at: row.vector_indexed_at ?? undefined, license: row.license ?? null, + vector_indexed_license: row.vector_indexed_license ?? null, }; } @@ -150,7 +151,7 @@ router.put('/api/repositories', (req, res) => { owner_login, owner_avatar_url, topics, ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, - subscribed_to_releases, vector_indexed_at, license + subscribed_to_releases, vector_indexed_at, license, vector_indexed_license ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name = excluded.name, @@ -178,7 +179,14 @@ router.put('/api/repositories', (req, res) => { last_edited = CASE WHEN excluded.last_edited IS NOT NULL AND excluded.last_edited != '' THEN excluded.last_edited ELSE repositories.last_edited END, subscribed_to_releases = excluded.subscribed_to_releases, vector_indexed_at = excluded.vector_indexed_at, - license = excluded.license + -- vector_indexed_license 是「上次向量索引时采用的 license」的照实记录， + -- 仅由索引流程（PATCH）写入；批量 upsert/sync 一律保留已存储值，避免被 + -- 同步流写入的当前 license 覆盖而破坏增量变更检测。 + vector_indexed_license = COALESCE(repositories.vector_indexed_license, excluded.vector_indexed_license), + -- 区分「省略 license 字段」与「显式提供 null/对象」： + -- 旧客户端/旧备份不含 license 字段时（@licenseProvided = 0）保留已存储值； + -- 显式提供时（@licenseProvided = 1）采用归一化后的 excluded.license（含 null 清空）。 + license = CASE WHEN @licenseProvided IS 1 THEN excluded.license ELSE repositories.license END `); const deleteAllReleases = db.prepare('DELETE FROM releases'); @@ -210,6 +218,8 @@ router.put('/api/repositories', (req, res) => { let count = 0; for (const repo of repositories) { const owner = repo.owner as { login?: string; avatar_url?: string } | undefined; + // 仅当 payload 显式提供 license 字段时才覆盖已存储值；省略（旧客户端/旧备份）则保留。 + const licenseProvided = Object.prototype.hasOwnProperty.call(repo, 'license') ? 1 : 0; stmt.run( repo.id, repo.name, repo.full_name, repo.description ?? null, repo.html_url, repo.stargazers_count ?? 0, repo.language ?? null, @@ -226,7 +236,11 @@ router.put('/api/repositories', (req, res) => { repo.custom_category ?? null, (repo.category_locked === true || repo.category_locked === 1) ? 1 : 0, repo.last_edited ?? null, (repo.subscribed_to_releases === true || repo.subscribed_to_releases === 1) ? 1 : 0, repo.vector_indexed_at ?? null, - toLicenseSpdxId(repo.license) + toLicenseSpdxId(repo.license), + // 备份中 vector_indexed_license 已是规范化的 SPDX id 字符串或 null； + // 非 string 一律清空，避免奇怪类型破坏增量谓词的字符串比较。 + typeof repo.vector_indexed_license === 'string' ? repo.vector_indexed_license || null : null, + { licenseProvided } ); count++; } @@ -263,6 +277,9 @@ router.patch('/api/repositories/:id', (req, res) => { // 规范化：null/undefined/空字符串 → null；仅接受字符串（ISO 时间戳） vector_indexed_at: (v) => (v === null || v === undefined || v === '') ? null : v, + // vector_indexed_license：规范化为 SPDX id 字符串或 null；非字符串一律清空。 + vector_indexed_license: (v) => + (v === null || v === undefined || v === '' || typeof v !== 'string') ? null : v, description: (v) => v, name: (v) => v, }; diff --git a/server/src/routes/sync.ts b/server/src/routes/sync.ts index 61bfa4939..f64fd5451 100644 --- a/server/src/routes/sync.ts +++ b/server/src/routes/sync.ts @@ -108,7 +108,7 @@ router.post('/api/sync/import', (req, res) => { owner_login, owner_avatar_url, topics, ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, - subscribed_to_releases, vector_indexed_at, license + subscribed_to_releases, vector_indexed_at, license, vector_indexed_license ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); for (const r of repos) { @@ -116,16 +116,19 @@ router.post('/api/sync/import', (req, res) => { if (!r.id || typeof r.id !== 'number') { throw new Error(`Invalid repository data: missing or invalid id`); } - // license：兼容旧备份（无该列→null）、GitHub 对象形态、已规范化的 SPDX 字符串 + // license：兼容旧备份（无该列→null）、GitHub 对象形态、已规范化的 SPDX 字符串。 + // 仅当候选字段在运行时确为字符串时才采用，避免 malformed 值（如 spdx_id: 123） + // 使 licenseValue 变成非字符串、污染导入事务；否则归一化为 SPDX id 或 null。 const rawLicense = (r as Record).license; - const licenseValue = - typeof rawLicense === 'string' - ? rawLicense || null - : rawLicense && typeof rawLicense === 'object' - ? (rawLicense as { spdx_id?: string; key?: string }).spdx_id || - (rawLicense as { key?: string }).key || - null - : null; + let licenseValue: string | null = null; + if (typeof rawLicense === 'string') { + licenseValue = rawLicense || null; + } else if (rawLicense && typeof rawLicense === 'object') { + const obj = rawLicense as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof obj.spdx_id === 'string' ? obj.spdx_id : null; + const key = typeof obj.key === 'string' ? obj.key : null; + licenseValue = spdx || key || null; + } repoStmt.run( r.id, r.name, r.full_name, r.description ?? null, r.html_url, r.stargazers_count ?? 0, r.language ?? null, @@ -142,7 +145,11 @@ router.post('/api/sync/import', (req, res) => { r.custom_category ?? null, (r.category_locked === true || r.category_locked === 1) ? 1 : 0, r.last_edited ?? null, r.subscribed_to_releases ? 1 : 0, r.vector_indexed_at ?? null, - licenseValue + licenseValue, + // vector_indexed_license：仅接受字符串（已是规范化 SPDX id），否则 null。 + // INSERT OR REPLACE 会整行替换，故备份无此列时落 null（影响：增量谓词会 + // 触发一次重索引回填指纹），合预期。 + typeof r.vector_indexed_license === 'string' ? r.vector_indexed_license || null : null ); } counts.repositories = repos.length; diff --git a/src/components/SearchBar.tsx b/src/components/SearchBar.tsx index f8cb19d74..c8f8a38a9 100644 --- a/src/components/SearchBar.tsx +++ b/src/components/SearchBar.tsx @@ -258,7 +258,7 @@ export const SearchBar: React.FC = () => { // Search helpers are intentionally kept as local closures; the explicit deps below // cover the state they read without causing a search loop on every render. // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchFilters.languages, searchFilters.tags, searchFilters.platforms, searchFilters.isAnalyzed, searchFilters.isSubscribed, searchFilters.isEdited, searchFilters.isCategoryLocked, searchFilters.analysisFailed, searchFilters.minStars, searchFilters.maxStars, searchFilters.sortBy, searchFilters.sortOrder, searchFilters.query, repositories, releaseSubscriptions, allCategories]); + }, [searchFilters.languages, searchFilters.tags, searchFilters.platforms, searchFilters.licenses, searchFilters.isAnalyzed, searchFilters.isSubscribed, searchFilters.isEdited, searchFilters.isCategoryLocked, searchFilters.analysisFailed, searchFilters.minStars, searchFilters.maxStars, searchFilters.sortBy, searchFilters.sortOrder, searchFilters.query, repositories, releaseSubscriptions, allCategories]); // Real-time search effect for repository name matching useEffect(() => { diff --git a/src/components/settings/VectorSearchSettings.tsx b/src/components/settings/VectorSearchSettings.tsx index 3857d22dd..a36d4209a 100644 --- a/src/components/settings/VectorSearchSettings.tsx +++ b/src/components/settings/VectorSearchSettings.tsx @@ -25,6 +25,7 @@ import { } from '../../services/vectorSearchService'; import { GitHubApiService } from '../../services/githubApi'; import type { EmbeddingApiType, EmbeddingConfig } from '../../types'; +import { normalizeLicense } from '../../utils/licenseFilter'; interface VectorSearchSettingsProps { t: (zh: string, en: string) => string; @@ -232,7 +233,10 @@ export const VectorSearchSettings: React.FC = ({ t }) .filter((t): t is string => !!t) .sort() .pop() || ''; - return contentTime > r.vector_indexed_at; + if (contentTime > r.vector_indexed_at) return true; + // license 变化（归一化后比对）同样需要重新索引：GitHub 同步可能只更新 license + // 而 last_edited/analyzed_at 不变，否则向量元数据中的 license 会过时。 + return normalizeLicense(r.vector_indexed_license ?? null) !== normalizeLicense(r.license ?? null); }).length; const indexableCount = repositories.filter((r) => r.analyzed_at && !r.analysis_failed).length; @@ -279,15 +283,26 @@ export const VectorSearchSettings: React.FC = ({ t }) try { // 每次点击时读取最新的 repositories，避免闭包捕获过期数据 const currentRepos = useAppStore.getState().repositories; + const now = new Date().toISOString(); + // 按 id 取 license，用于在 stamp vector_indexed_at 的同时记录本次采用的 + // 归一化 license（向量增量谓词据此判断 license 变更触发重索引）。 + const licenseById = new Map(currentRepos.map(r => [r.id, r.license ?? null])); + const stampRepo = (id: number) => ({ + id, + patch: { vector_indexed_at: now, vector_indexed_license: normalizeLicense(licenseById.get(id) ?? null) }, + }); // 1. 清除所有 vector_indexed_at（包括之前失败/不可索引的 repo 的残留值） // 用 updateRepositoriesMetadata 避免重置当前过滤的 searchResults + // 同步清除 vector_indexed_license，使 license 指纹与 stamp 同进退。 updateRepositoriesMetadata( - currentRepos.filter(r => r.vector_indexed_at).map(r => ({ id: r.id, patch: { vector_indexed_at: undefined } })) + currentRepos.filter(r => r.vector_indexed_at).map(r => ({ + id: r.id, + patch: { vector_indexed_at: undefined, vector_indexed_license: undefined }, + })) ); // 2. 全量索引，逐批确认后立即 stamp（中断不丢失已确认进度） - const now = new Date().toISOString(); const stampedRepoIds: number[] = []; const result = await indexAllRepos(currentRepos, clients.embeddingClient, clients.vectorService, { onProgress: (progress) => setVectorIndexingState({ @@ -305,14 +320,14 @@ export const VectorSearchSettings: React.FC = ({ t }) // 批量 stamp：每 32 个（一个 batch）刷新一次，减少 UI 刷新频率 if (stampedRepoIds.length % 32 === 0) { const batch = stampedRepoIds.splice(0, stampedRepoIds.length); - updateRepositoriesMetadata(batch.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(batch.map(stampRepo)); } }, }); // stamp 剩余未刷新的 if (stampedRepoIds.length > 0) { - updateRepositoriesMetadata(stampedRepoIds.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(stampedRepoIds.map(stampRepo)); } // 3. cleanup：全量重建后只保留本次成功重建的向量 @@ -368,6 +383,13 @@ export const VectorSearchSettings: React.FC = ({ t }) ); const now = new Date().toISOString(); + // 与全量重建一致：stamp 时同步记录本次索引采用的归一化 license， + // 供增量谓词下次判断 license 是否变化。 + const licenseById = new Map(currentRepos.map(r => [r.id, r.license ?? null])); + const stampRepo = (id: number) => ({ + id, + patch: { vector_indexed_at: now, vector_indexed_license: normalizeLicense(licenseById.get(id) ?? null) }, + }); const stampedRepoIds: number[] = []; const result = await indexAllRepos(currentRepos, clients.embeddingClient, clients.vectorService, { onProgress: (progress) => setVectorIndexingState({ @@ -386,14 +408,14 @@ export const VectorSearchSettings: React.FC = ({ t }) stampedRepoIds.push(repoId); if (stampedRepoIds.length % 32 === 0) { const batch = stampedRepoIds.splice(0, stampedRepoIds.length); - updateRepositoriesMetadata(batch.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(batch.map(stampRepo)); } }, }); // stamp 剩余未刷新的 if (stampedRepoIds.length > 0) { - updateRepositoriesMetadata(stampedRepoIds.map(id => ({ id, patch: { vector_indexed_at: now } }))); + updateRepositoriesMetadata(stampedRepoIds.map(stampRepo)); } setVectorIndexingState({ result, isIndexing: false, phase: null }); @@ -427,7 +449,8 @@ export const VectorSearchSettings: React.FC = ({ t }) .filter((t): t is string => !!t) .sort() .pop() || ''; - return contentTime > r.vector_indexed_at; + if (contentTime > r.vector_indexed_at) return true; + return normalizeLicense(r.vector_indexed_license ?? null) !== normalizeLicense(r.license ?? null); }).length; const skippedCount = currentRepos.length - attemptedCount; setVectorIndexingState({ diff --git a/src/services/aiService.test.ts b/src/services/aiService.test.ts new file mode 100644 index 000000000..e3e813da8 --- /dev/null +++ b/src/services/aiService.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import type { Repository } from '../types'; +import { AIService } from './aiService'; + +// Minimal AIConfig that lets AIService construct without a real token. +const makeConfig = () => ({ + id: 'test', + name: 'test', + apiType: 'openai' as const, + baseUrl: 'http://localhost:0', + apiKey: '', + model: 'gpt-test', + isActive: true, +}); + +function makeRepo(partial: Partial & Pick): Repository { + return { + description: null, + html_url: `https://github.com/${partial.full_name}`, + stargazers_count: 0, + forks_count: 0, + forks: 0, + language: 'TypeScript', + created_at: '2024-01-01T00:00:00Z', + updated_at: '2024-06-01T00:00:00Z', + pushed_at: '2024-06-01T00:00:00Z', + owner: { login: 'owner', avatar_url: '' }, + topics: [], + ...partial, + }; +} + +describe('AIService.searchRepositoriesWithReranking — enhanced basic search fallback', () => { + beforeEach(() => { + // Force the AI request path to fail so we fall back to performEnhancedBasicSearch. + (window.fetch as ReturnType).mockImplementation(() => { + throw new Error('network disabled in test'); + }); + }); + + it('ranks a license-matching repo above a higher-star non-matching repo when the query mixes license + other terms', async () => { + // A matches both the name term ("react") and the license term ("mit"). + const repoA = makeRepo({ + id: 1, + name: 'react-app', + full_name: 'acme/react-app', + stargazers_count: 500, + license: 'MIT', + }); + // B has far more stars and matches the name term, but NOT the license term. + const repoB = makeRepo({ + id: 2, + name: 'react-lib', + full_name: 'acme/react-lib', + stargazers_count: 1000, + license: 'Apache-2.0', + }); + + const service = new AIService(makeConfig() as never, 'en'); + const results = await service.searchRepositoriesWithReranking([repoA, repoB], 'react mit'); + + const ids = results.map((r) => r.id); + // With the license weight, A's license match outweighs B's popularity edge. + expect(ids).toEqual([1, 2]); + expect(results).toHaveLength(2); + }); +}); diff --git a/src/services/aiService.ts b/src/services/aiService.ts index 3918553b8..712b8bf51 100644 --- a/src/services/aiService.ts +++ b/src/services/aiService.ts @@ -1350,6 +1350,9 @@ ${repoInfo} // Platform and language matches if (searchableFields.aiPlatforms.includes(word)) score += 0.18; if (searchableFields.language.includes(word)) score += 0.12; + + // License matches + if (searchableFields.license.includes(word)) score += 0.2; }); // Boost for exact matches diff --git a/src/services/githubApi.ts b/src/services/githubApi.ts index 5db30f3c4..c109956dd 100644 --- a/src/services/githubApi.ts +++ b/src/services/githubApi.ts @@ -601,7 +601,10 @@ export class GitHubApiService { const endpoint = username ? `/users/${encodeURIComponent(username)}/subscriptions?page=${page}&per_page=${perPage}` : `/user/subscriptions?page=${page}&per_page=${perPage}`; - return this.makeRequest(endpoint); + // GitHub 对 watched repos 同样返回原始 license 对象，这里统一归一化为 SPDX id / null， + // 与 /user/starred 路径保持一致，避免下游 normalizeLicense 拿到对象时崩溃。 + const repos = await this.makeRequest(endpoint); + return repos.map((repo) => ({ ...repo, license: toLicenseSpdxId(repo.license) })); } async getAllWatchedRepositories(username?: string): Promise { diff --git a/src/services/vectorSearchService.test.ts b/src/services/vectorSearchService.test.ts index d0ff9f135..3b9d79cbd 100644 --- a/src/services/vectorSearchService.test.ts +++ b/src/services/vectorSearchService.test.ts @@ -138,6 +138,94 @@ describe('indexAllRepos incremental filtering', () => { }); }); +describe('indexAllRepos incremental filtering — license change invalidation', () => { + it('reindexes an indexed repo whose license changed even when content timestamps are unchanged', async () => { + // Repo was indexed under MIT; its vector_indexed_at is newer than content, so without + // the license fingerprint check it would be skipped. GitHub sync then changes the + // license to Apache-2.0 — the incremental predicate must reindex to refresh metadata. + const repos = [ + makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: 'MIT', + license: 'Apache-2.0', + analyzed_at: '2026-01-04T00:00:00.000Z', + // No last_edited / newer analyzed_at => content time predates vector_indexed_at. + }), + ]; + + const client = makeIndexClient(); + const vectorService = makeVectorService(); + const indexedRepoIds: number[] = []; + const result = await indexAllRepos(repos, client, vectorService, { + incremental: true, + formatVersion: EMBEDDING_FORMAT_VERSION, + currentFormatVersion: EMBEDDING_FORMAT_VERSION, + indexMode: 'description', + onRepoIndexed: (repoId) => indexedRepoIds.push(repoId), + }); + + // License differs from the indexed fingerprint => reindex, despite no content-time change. + expect(indexedRepoIds).toEqual([1]); + expect(result.indexedRepoIds).toEqual([1]); + }); + + it('treats a null-vs-no-assertion drift as a license change and reindexes', async () => { + // normalizeLicense collapses both null and 'NOASSERTION' to the sentinel, so an indexed + // repo whose stored license was null (fingerprint not yet captured) must reindex once to + // capture the fingerprint, then settle on subsequent runs. + const repos = [ + makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: null, + license: 'MIT', + analyzed_at: '2026-01-04T00:00:00.000Z', + }), + ]; + + const client = makeIndexClient(); + const vectorService = makeVectorService(); + const indexedRepoIds: number[] = []; + const result = await indexAllRepos(repos, client, vectorService, { + incremental: true, + formatVersion: EMBEDDING_FORMAT_VERSION, + currentFormatVersion: EMBEDDING_FORMAT_VERSION, + indexMode: 'description', + onRepoIndexed: (repoId) => indexedRepoIds.push(repoId), + }); + + // null fingerprint vs MIT license => mismatch => reindex. + expect(indexedRepoIds).toEqual([1]); + expect(result.indexedRepoIds).toEqual([1]); + }); + + it('skips an indexed repo whose license is unchanged', async () => { + const repos = [ + makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: 'MIT', + license: 'MIT', + analyzed_at: '2026-01-04T00:00:00.000Z', + }), + ]; + + const client = makeIndexClient(); + const vectorService = makeVectorService(); + const indexedRepoIds: number[] = []; + const result = await indexAllRepos(repos, client, vectorService, { + incremental: true, + formatVersion: EMBEDDING_FORMAT_VERSION, + currentFormatVersion: EMBEDDING_FORMAT_VERSION, + indexMode: 'description', + onRepoIndexed: (repoId) => indexedRepoIds.push(repoId), + }); + + // No content change and license fingerprint matches => nothing to index. + expect(indexedRepoIds).toEqual([]); + expect(result.indexedRepoIds).toEqual([]); + expect(result.indexed).toBe(0); + }); +}); + describe('embedWithFallback', () => { it('uses fast batch path on success', async () => { const client = makeClient(async (texts) => texts.map((_, i) => vec(i))); diff --git a/src/services/vectorSearchService.ts b/src/services/vectorSearchService.ts index 8043f2df4..323ed5942 100644 --- a/src/services/vectorSearchService.ts +++ b/src/services/vectorSearchService.ts @@ -6,6 +6,7 @@ */ import type { EmbeddingConfig, VectorSearchConfig, Repository } from '../types'; +import { normalizeLicense } from '../utils/licenseFilter'; // ============================================================ // EmbeddingClient @@ -522,7 +523,11 @@ export async function indexAllRepos( .filter((t): t is string => !!t) .sort() .pop() || ''; - return contentTime > r.vector_indexed_at; + if (contentTime > r.vector_indexed_at) return true; + // license 变化（归一化后比对）：GitHub 同步可能仅更新 license 而 last_edited 不变， + // 此时向量元数据与嵌入文本中的 License 字段会过时，故需重新索引。 + const indexedLicense = r.vector_indexed_license ?? null; + return normalizeLicense(indexedLicense) !== normalizeLicense(r.license ?? null); }); } let indexed = 0; diff --git a/src/types/index.ts b/src/types/index.ts index fe61a40f0..a6bb959d2 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -30,6 +30,8 @@ export interface Repository { category_locked?: boolean; last_edited?: string; vector_indexed_at?: string; // ISO timestamp of last successful vector indexing + /** 上次向量索引时采用的 license（SPDX id / null）。增量谓词据此判断 license 是否变化以触发重索引。 */ + vector_indexed_license?: string | null; last_release_fetch_time?: string; // ISO timestamp, for incremental sync has_fetched_releases?: boolean; // whether this repo has been synced for releases /** SPDX id（如 'MIT'、'Apache-2.0'）；无许可证/未识别为 null。AI/搜索/过滤均以此为准。 */ diff --git a/src/utils/repoSearch.test.ts b/src/utils/repoSearch.test.ts index 7ef1489a6..109543d44 100644 --- a/src/utils/repoSearch.test.ts +++ b/src/utils/repoSearch.test.ts @@ -101,8 +101,20 @@ describe('applyRepoFilters', () => { }); it('aggregates no-license repos via the sentinel', () => { - // normalizeLicense collapses 'NOASSERTION' (and null/'Other') to the sentinel + // normalizeLicense collapses every no-license shape to the sentinel, so the + // "no license" UI bucket is a single stable key regardless of how GitHub + // reports the absence: null/'Other' (key-only licenses)/'NOASSERTION'/ + // 'none'/''/and mixed-case 'noassertion' all collapse to NO_LICENSE_SENTINEL. + expect(normalizeLicense(null)).toBe(NO_LICENSE_SENTINEL); expect(normalizeLicense('NOASSERTION')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('Other')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('none')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('noassertion')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense('NoAssertion')).toBe(NO_LICENSE_SENTINEL); + + // A repo whose stored license collapses to the sentinel is matched by the + // "no license" filter. (sample[2] carries license: 'NOASSERTION' → sentinel.) const hits = applyRepoFilters(sample, { licenses: [NO_LICENSE_SENTINEL] }); expect(hits.map((r) => r.id)).toEqual([3]); }); diff --git a/src/utils/repoSearch.ts b/src/utils/repoSearch.ts index a59fd4b80..afe99f343 100644 --- a/src/utils/repoSearch.ts +++ b/src/utils/repoSearch.ts @@ -28,7 +28,7 @@ export function performBasicTextSearch(repos: T[], query: ...(repo.ai_platforms || []), ...(repo.custom_tags || []), repo.custom_category || '', - repo.license || '', + normalizeLicense(repo.license), ] .join(' ') .toLowerCase(); From b0fad9e0ee16cd8dafd9461dd18d27384274cdc5 Mon Sep 17 00:00:00 2001 From: AmintaCCCP Date: Wed, 29 Jul 2026 22:59:15 +0800 Subject: [PATCH 3/7] fix: address follow-up CodeRabbit review on PR #254 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (Outside-diff, 🔴 Critical) repositories.ts bulk upsert: add the missing 28th VALUES placeholder for vector_indexed_license. Adding the column without its '?' made the placeholder count (27) mismatch the 28 positional args bound by stmt.run, which would throw "Too few parameter values" on every upsert. (Nitpick, 🔵 Trivial) vectorSearchService.ts: make license optional on VectorizeVector.metadata and VectorQueryResult.metadata. Pre-PR (embedding v2) vectors lack the license key in stored metadata until reindexed, so the typed `license: string` did not reflect runtime reality. Consumers already read metadata defensively (`r.metadata?.full_name || ''`), so this is a type-fidelity fix. (Nitpick, 🔵 Trivial) Extract the needs-reindex predicate into an exported needsReindex(repo, formatVersionChanged) helper — the same content-time + normalized-license fingerprint check was duplicated across indexAllRepos and two UI mirrors (unindexedCount, attemptedCount). All three now call the single shared helper, preventing future desync between what the UI displays and what indexAllRepos actually reindexes. Added a 6-test suite covering the helper's branches including the null/NOASSERTION sentinel-equivalence case. Tests: vitest 140 passed (adds 6 needsReindex tests). tsc clean for frontend and server. Co-Authored-By: Claude Opus 4.8 --- server/src/routes/repositories.ts | 2 +- .../settings/VectorSearchSettings.tsx | 24 +++------- src/services/vectorSearchService.test.ts | 44 +++++++++++++++++ src/services/vectorSearchService.ts | 48 ++++++++++++------- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/server/src/routes/repositories.ts b/server/src/routes/repositories.ts index 06e7b083b..13b3008b5 100644 --- a/server/src/routes/repositories.ts +++ b/server/src/routes/repositories.ts @@ -152,7 +152,7 @@ router.put('/api/repositories', (req, res) => { ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, subscribed_to_releases, vector_indexed_at, license, vector_indexed_license - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET name = excluded.name, full_name = excluded.full_name, diff --git a/src/components/settings/VectorSearchSettings.tsx b/src/components/settings/VectorSearchSettings.tsx index a36d4209a..12e4817f8 100644 --- a/src/components/settings/VectorSearchSettings.tsx +++ b/src/components/settings/VectorSearchSettings.tsx @@ -21,6 +21,7 @@ import { EmbeddingClient, VectorSearchService, indexAllRepos, + needsReindex, EMBEDDING_FORMAT_VERSION, } from '../../services/vectorSearchService'; import { GitHubApiService } from '../../services/githubApi'; @@ -225,18 +226,11 @@ export const VectorSearchSettings: React.FC = ({ t }) } }, [formWorkerUrl, formAuthToken, setVectorSearchStatus]); - // 未索引数量（已分析、未失败、未向量索引或内容已更新） + // 未索引数量（已分析、未失败、未向量索引或内容已更新）。仅计算内容/license 维度， + // 不含格式版本升级——后者由 incrementalTargetCount 在 formatVersionNeedsReindex 时另行叠加。 const unindexedCount = repositories.filter((r) => { if (!r.analyzed_at || r.analysis_failed) return false; - if (!r.vector_indexed_at) return true; - const contentTime = [r.last_edited, r.analyzed_at] - .filter((t): t is string => !!t) - .sort() - .pop() || ''; - if (contentTime > r.vector_indexed_at) return true; - // license 变化（归一化后比对）同样需要重新索引：GitHub 同步可能只更新 license - // 而 last_edited/analyzed_at 不变，否则向量元数据中的 license 会过时。 - return normalizeLicense(r.vector_indexed_license ?? null) !== normalizeLicense(r.license ?? null); + return needsReindex(r, false); }).length; const indexableCount = repositories.filter((r) => r.analyzed_at && !r.analysis_failed).length; @@ -442,15 +436,11 @@ export const VectorSearchSettings: React.FC = ({ t }) } else { const msg = err instanceof Error ? err.message : String(err); const currentRepos = useAppStore.getState().repositories; + // 与 indexAllRepos 增量谓词保持一致（含格式版本升级判定），避免计数漂移。 + const formatVersionChanged = currentEmbeddingFormatVersion < EMBEDDING_FORMAT_VERSION; const attemptedCount = currentRepos.filter((r) => { if (!r.analyzed_at || r.analysis_failed) return false; - if (!r.vector_indexed_at) return true; - const contentTime = [r.last_edited, r.analyzed_at] - .filter((t): t is string => !!t) - .sort() - .pop() || ''; - if (contentTime > r.vector_indexed_at) return true; - return normalizeLicense(r.vector_indexed_license ?? null) !== normalizeLicense(r.license ?? null); + return needsReindex(r, formatVersionChanged); }).length; const skippedCount = currentRepos.length - attemptedCount; setVectorIndexingState({ diff --git a/src/services/vectorSearchService.test.ts b/src/services/vectorSearchService.test.ts index 3b9d79cbd..f717bf4d0 100644 --- a/src/services/vectorSearchService.test.ts +++ b/src/services/vectorSearchService.test.ts @@ -5,6 +5,7 @@ import { truncateForRetry, embedWithFallback, indexAllRepos, + needsReindex, EMBEDDING_FORMAT_VERSION, } from './vectorSearchService'; @@ -51,6 +52,49 @@ const makeVectorService = () => ({ const LENGTH_ERROR = () => new Error('Embedding API error 400: {"code":20015,"message":"The parameter is invalid.","data":null}'); +describe('needsReindex', () => { + // 共享：未索引 / 内容更新 / license 变化 / 格式升级——权威谓词，UI 与 indexAllRepos 共用。 + const indexed = (overrides: Partial = {}): Repository => makeRepository(1, { + vector_indexed_at: '2026-01-10T00:00:00.000Z', + vector_indexed_license: 'MIT', + license: 'MIT', + analyzed_at: '2026-01-04T00:00:00.000Z', + ...overrides, + }); + + it('needs reindex when a repo was never indexed', () => { + expect(needsReindex({ ...indexed(), vector_indexed_at: undefined }, false)).toBe(true); + }); + + it('needs reindex when content time is newer than vector_indexed_at', () => { + expect(needsReindex(indexed({ last_edited: '2026-01-12T00:00:00.000Z' }), false)).toBe(true); + }); + + it('needs reindex when license changed (normalized drift)', () => { + expect(needsReindex(indexed({ license: 'Apache-2.0' }), false)).toBe(true); + }); + + it('needs reindex when format version changed regardless of content', () => { + expect(needsReindex(indexed(), true)).toBe(true); + }); + + it('does NOT need reindex when content and license are unchanged', () => { + expect(needsReindex(indexed(), false)).toBe(false); + }); + + it('treats null vs NOASSERTION sentinel as equal (no reindex)', () => { + // Both normalize to NO_LICENSE_SENTINEL, so null fingerprint ↔ no-assertion license is a match. + expect(needsReindex( + indexed({ vector_indexed_license: null, license: null }), + false, + )).toBe(false); + expect(needsReindex( + indexed({ vector_indexed_license: null, license: 'NOASSERTION' }), + false, + )).toBe(false); + }); +}); + describe('looksLikeLengthError', () => { it('detects SiliconFlow 20015 length errors', () => { expect(looksLikeLengthError(LENGTH_ERROR())).toBe(true); diff --git a/src/services/vectorSearchService.ts b/src/services/vectorSearchService.ts index 323ed5942..e8ac3a4e2 100644 --- a/src/services/vectorSearchService.ts +++ b/src/services/vectorSearchService.ts @@ -188,7 +188,9 @@ export interface VectorizeVector { language: string; stars: number; tags: string[]; - license: string; + // 可选：此 PR 之前（embedding v2）索引的向量不含 license 键，重索引前以 undefined 出现。 + // indexAllRepos 写入时会填入字符串；消费方请按可选处理。 + license?: string; }; } @@ -201,7 +203,8 @@ export interface VectorQueryResult { language: string; stars: number; tags: string[]; - license: string; + // 同上：旧向量查询返回时缺该键，按可选处理。 + license?: string; }; } @@ -377,6 +380,32 @@ export function buildEmbeddingText(repo: Repository, readmeContent?: string, max return parts.join('\n'); } +/** + * 判断单个仓库是否需要重新向量索引（增量谓词的权威实现）。 + * + * 以下任一成立即视为需要重索引： + * - 从未索引（`vector_indexed_at` 缺失）； + * - embedding 格式版本升级（由调用方据 `formatVersionChanged` 传入）； + * - 内容时间（`last_edited` 与 `analyzed_at` 中较新者）晚于上次索引时间； + * - license 变化（归一化后比对 `vector_indexed_license` 与 `license`）。 + * + * 该模块与 `VectorSearchSettings.tsx` 的 `unindexedCount` / `attemptedCount` 谓词 + * 必须保持一致，故统一抽取为可选导出供复用，避免三处副本分别漂移。 + */ +export function needsReindex(repo: Pick, formatVersionChanged: boolean): boolean { + if (!repo.vector_indexed_at) return true; // 从未索引 + if (formatVersionChanged) return true; // 格式版本升级，需要重新索引 + // 取 last_edited 与 analyzed_at 中较新者作为内容时间，更新后需要重新索引 + const contentTime = [repo.last_edited, repo.analyzed_at] + .filter((t): t is string => !!t) + .sort() + .pop() || ''; + if (contentTime > repo.vector_indexed_at) return true; + // license 变化（归一化后比对）：GitHub 同步可能仅更新 license 而 last_edited 不变， + // 此时向量元数据与嵌入文本中的 License 字段会过时，故需重新索引。 + return normalizeLicense(repo.vector_indexed_license ?? null) !== normalizeLicense(repo.license ?? null); +} + /** * 全量/增量重建向量索引 * 遍历已分析仓库，分批生成 embedding 并 upsert 到 Worker @@ -515,20 +544,7 @@ export async function indexAllRepos( // 嵌入文本格式版本变化时，强制重新索引所有向量以避免混合格式 // 缺失版本号视为 v1（旧格式），仍需触发升级 const formatVersionChanged = (options.formatVersion ?? 1) < (options.currentFormatVersion ?? EMBEDDING_FORMAT_VERSION); - indexable = indexable.filter((r) => { - if (!r.vector_indexed_at) return true; // 从未索引 - if (formatVersionChanged) return true; // 格式版本升级，需要重新索引 - // 取 last_edited 与 analyzed_at 中较新者作为内容时间，更新后需要重新索引 - const contentTime = [r.last_edited, r.analyzed_at] - .filter((t): t is string => !!t) - .sort() - .pop() || ''; - if (contentTime > r.vector_indexed_at) return true; - // license 变化（归一化后比对）：GitHub 同步可能仅更新 license 而 last_edited 不变， - // 此时向量元数据与嵌入文本中的 License 字段会过时，故需重新索引。 - const indexedLicense = r.vector_indexed_license ?? null; - return normalizeLicense(indexedLicense) !== normalizeLicense(r.license ?? null); - }); + indexable = indexable.filter((r) => needsReindex(r, formatVersionChanged)); } let indexed = 0; let errors = 0; From 882d3b818ce67da50e2adf48728f0599a3e45f9a Mon Sep 17 00:00:00 2001 From: AmintaCCCP Date: Wed, 29 Jul 2026 23:18:50 +0800 Subject: [PATCH 4/7] fix: stop client crash "e.toLowerCase is not a function" on non-string license MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: normalizeLicense assumed its argument was a string and called v.toLowerCase() unconditionally. A repo whose license never went through toLicenseSpdxId can carry a raw GitHub license object { key, spdx_id, name, url } — legacy persisted store, third-party backup import, or GitHub API returns not yet normalized (only /user/starred and getWatchedRepositories are). On render, performBasicTextSearch / performEnhancedBasicSearch / RepositoryCard invoke normalizeLicense(repo.license), where the object had no .toLowerCase → crash. aiService additionally did (repo.license || '').toLowerCase(), the same failure. Fix: - normalizeLicense(v: unknown) is now defensive across all 3 mirrors (src/utils/licenseFilter.ts, server/src/mcp/repoSearch.ts, electron/mcpLocalServer.js): GitHub-shaped objects resolve to spdx_id ?? key, then sentinel-normalize; any other non-string collapes to the no-license sentinel; never call .toLowerCase() on a non-string. - aiService.performEnhancedBasicSearch/BasicSearch/static searchRepositories and vectorSearchService metadata construction route license through normalizeLicense instead of raw .toLowerCase() / `repo.license || ''`, so object licenses both stop crashing and stop polluting the searchable document / vector metadata with "[object Object]". Regression coverage: - licenseFilter.test.ts: object (spdx_id/key), object missing fields, number, boolean, sentinel-pass-through — all return sentinel or resolved string, no throw. - aiService.test.ts: searchRepositoriesWithReranking with a raw GitHub license object resolves to 'MIT' and ranks — previously threw. Tests: vitest 142 passed. tsc clean for frontend and server. Co-Authored-By: Claude Opus 4.8 --- electron/mcpLocalServer.js | 10 +++++++++- server/src/mcp/repoSearch.ts | 13 +++++++++++-- src/services/aiService.test.ts | 19 +++++++++++++++++++ src/services/aiService.ts | 9 +++++---- src/services/vectorSearchService.ts | 4 +++- src/utils/licenseFilter.test.ts | 22 ++++++++++++++++++++++ src/utils/licenseFilter.ts | 21 ++++++++++++++++++--- 7 files changed, 87 insertions(+), 11 deletions(-) diff --git a/electron/mcpLocalServer.js b/electron/mcpLocalServer.js index 2bc37c713..8e32ac9ca 100644 --- a/electron/mcpLocalServer.js +++ b/electron/mcpLocalServer.js @@ -35,7 +35,15 @@ function performBasicTextSearch(repos, query) { const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); function normalizeLicense(v) { - if (!v) return NO_LICENSE_SENTINEL; + if (v == null || v === '') return NO_LICENSE_SENTINEL; + if (typeof v === 'object') { + const spdx = typeof v.spdx_id === 'string' ? v.spdx_id : null; + const key = typeof v.key === 'string' ? v.key : null; + const resolved = spdx || key; + if (!resolved) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; + } + if (typeof v !== 'string') return NO_LICENSE_SENTINEL; return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; } diff --git a/server/src/mcp/repoSearch.ts b/server/src/mcp/repoSearch.ts index 8aa5679cd..0b267e560 100644 --- a/server/src/mcp/repoSearch.ts +++ b/server/src/mcp/repoSearch.ts @@ -9,8 +9,17 @@ */ export const NO_LICENSE_SENTINEL = '__NO_LICENSE__'; const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); -export function normalizeLicense(v: string | null | undefined): string { - if (!v) return NO_LICENSE_SENTINEL; +export function normalizeLicense(v: unknown): string { + if (v == null || v === '') return NO_LICENSE_SENTINEL; + if (typeof v === 'object') { + const l = v as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id : null; + const key = typeof l.key === 'string' ? l.key : null; + const resolved = spdx ?? key; + if (!resolved) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; + } + if (typeof v !== 'string') return NO_LICENSE_SENTINEL; return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; } diff --git a/src/services/aiService.test.ts b/src/services/aiService.test.ts index e3e813da8..0a8ca2d58 100644 --- a/src/services/aiService.test.ts +++ b/src/services/aiService.test.ts @@ -64,4 +64,23 @@ describe('AIService.searchRepositoriesWithReranking — enhanced basic search fa expect(ids).toEqual([1, 2]); expect(results).toHaveLength(2); }); + + it('does not crash when a repo carries a raw GitHub license object (toLowerCase defensive)', async () => { + // Regression for "e.toLowerCase is not a function": a repo whose license never passed + // through toLicenseSpdxId (legacy persisted store / third-party import) keeps a raw + // GitHub object `{ key, spdx_id, ... }`. performEnhancedBasicSearch must reduce it via + // normalizeLicense rather than (repo.license || '').toLowerCase(). + const repoA = makeRepo({ + id: 3, + name: 'react-legacy', + full_name: 'acme/react-legacy', + stargazers_count: 10, + license: { spdx_id: 'MIT', key: 'MIT', name: 'MIT License', url: 'https://api.github.com/licenses/mit' } as never, + }); + + const service = new AIService(makeConfig() as never, 'en'); + // Should resolve the license object to 'MIT' and rank the repo — not throw. + const results = await service.searchRepositoriesWithReranking([repoA], 'mit'); + expect(results.map((r) => r.id)).toEqual([3]); + }); }); diff --git a/src/services/aiService.ts b/src/services/aiService.ts index 712b8bf51..73e466086 100644 --- a/src/services/aiService.ts +++ b/src/services/aiService.ts @@ -1,6 +1,7 @@ import { Repository, Gist, AIConfig, AIApiType } from '../types'; import { backend } from './backendAdapter'; import { buildApiUrl, buildFinalApiUrl } from '../utils/apiUrlBuilder'; +import { normalizeLicense } from '../utils/licenseFilter'; import { logger } from './logger'; interface OpenAIResponseContentPart { @@ -1317,7 +1318,7 @@ ${repoInfo} aiPlatforms: (repo.ai_platforms || []).join(' ').toLowerCase(), customDescription: (repo.custom_description || '').toLowerCase(), customTags: (repo.custom_tags || []).join(' ').toLowerCase(), - license: (repo.license || '').toLowerCase(), + license: normalizeLicense(repo.license).toLowerCase(), }; // Check if any query word matches any field @@ -1439,7 +1440,7 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), - repo.license || '', + normalizeLicense(repo.license), ].join(' ').toLowerCase(); // Check if any of the AI-enhanced terms match @@ -1465,7 +1466,7 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), - repo.license || '', + normalizeLicense(repo.license), ].join(' ').toLowerCase(); // Split query into words and check if all words are present @@ -1490,7 +1491,7 @@ Reply in JSON format: repo.ai_summary || '', ...(repo.ai_tags || []), ...(repo.ai_platforms || []), - repo.license || '', + normalizeLicense(repo.license), ].join(' ').toLowerCase(); // Split query into words and check if all words are present diff --git a/src/services/vectorSearchService.ts b/src/services/vectorSearchService.ts index e8ac3a4e2..ed3907d74 100644 --- a/src/services/vectorSearchService.ts +++ b/src/services/vectorSearchService.ts @@ -610,7 +610,9 @@ export async function indexAllRepos( language: batch[j].language || '', stars: batch[j].stargazers_count || 0, tags: batch[j].ai_tags || [], - license: batch[j].license || '', + // 历史/导入数据 license 可能是 GitHub 对象或数字；统一经 normalizeLicense + // 降为 SPDX id / 哨兵，避免把 "[object Object]" 写入向量元数据。 + license: normalizeLicense(batch[j].license), }, }); } else { diff --git a/src/utils/licenseFilter.test.ts b/src/utils/licenseFilter.test.ts index 33c76ee5c..1ecac1054 100644 --- a/src/utils/licenseFilter.test.ts +++ b/src/utils/licenseFilter.test.ts @@ -46,6 +46,28 @@ describe('normalizeLicense', () => { expect(normalizeLicense('other')).toBe(NO_LICENSE_SENTINEL); expect(normalizeLicense('none')).toBe(NO_LICENSE_SENTINEL); }); + + it('never throws on a non-string license (GitHub object / number / legacy data)', () => { + // Regression: performBasicTextSearch / performEnhancedBasicSearch / RepositoryCard call + // normalizeLicense(repo.license) during render. A repo that never passed through + // toLicenseSpdxId (legacy persisted store, third-party import, or a return path not + // yet normalized) can carry a raw GitHub license object { key, spdx_id, ... } or a + // number. normalizeLicense must reduce these without calling .toLowerCase() on a + // non-string and crashing the client render ("e.toLowerCase is not a function"). + // GitHub object → resolve SPDX id (spdx_id preferred over key): + expect(normalizeLicense({ spdx_id: 'MIT', key: 'MIT', name: 'MIT License', url: 'u' } as unknown)) + .toBe('MIT'); + // GitHub "Other" object (no SPDX): key 'Other' → sentinel. + expect(normalizeLicense({ key: 'Other', spdx_id: 'NOASSERTION', name: 'Other' } as unknown)) + .toBe(NO_LICENSE_SENTINEL); + // Object missing both string fields → sentinel. + expect(normalizeLicense({ name: 'Custom' } as unknown)).toBe(NO_LICENSE_SENTINEL); + // Non-string truthy values (number/boolean) → sentinel, no crash. + expect(normalizeLicense(123 as unknown)).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(true as unknown)).toBe(NO_LICENSE_SENTINEL); + // String already carrying the sentinel stays. + expect(normalizeLicense(NO_LICENSE_SENTINEL)).toBe(NO_LICENSE_SENTINEL); + }); }); describe('buildEmbeddingText license', () => { diff --git a/src/utils/licenseFilter.ts b/src/utils/licenseFilter.ts index c3c103fce..b71b868f3 100644 --- a/src/utils/licenseFilter.ts +++ b/src/utils/licenseFilter.ts @@ -26,10 +26,25 @@ const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-licens /** * 把仓库的 license 值归一化为「SPDX id」或「无 license 哨兵」。 * 比对大小写不敏感，以收敛历史备份/第三方源写入的小写变体（如 'other'、'none'）。 - * @param v 原始 license 值（SPDX id / 哨兵值 / null / undefined / 空串） + * + * 防御：`v` 可能并非字符串——历史持久化数据、第三方备份导入，或尚未走 {@link toLicenseSpdxId} + * 的 GitHub 原始对象 `{ key, spdx_id, name, url }` 都可能携带非字符串 license。此处不再假设字符串： + * 对象形态先还原为 `spdx_id ?? key`（再字符串比对），其余非字符串一律按「无 license」归并， + * 避免对对象/数字调用 `.toLowerCase()` 导致客户端渲染崩溃。 + * @param v 原始 license 值（SPDX id / GitHub 对象 / 数字 / null / undefined / 空串） * @returns 归一化后的字符串；无 license 时返回 {@link NO_LICENSE_SENTINEL} */ -export function normalizeLicense(v: string | null | undefined): string { - if (!v) return NO_LICENSE_SENTINEL; +export function normalizeLicense(v: unknown): string { + if (v == null || v === '') return NO_LICENSE_SENTINEL; + if (typeof v === 'object') { + // GitHub license 对象：优先 spdx_id（如 'MIT'），回退 key（如 'Other'） + const l = v as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id : null; + const key = typeof l.key === 'string' ? l.key : null; + const resolved = spdx ?? key; + if (!resolved) return NO_LICENSE_SENTINEL; + return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; + } + if (typeof v !== 'string') return NO_LICENSE_SENTINEL; // 数字等为非合法 license return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; } From 2a93dd7f7a97ee274e01ee101355ba36b83242b5 Mon Sep 17 00:00:00 2001 From: AmintaCCCP Date: Wed, 29 Jul 2026 23:43:06 +0800 Subject: [PATCH 5/7] fix: normalizeLicense falls back from blank spdx_id to a valid key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit r3675689914: `{ spdx_id: '', key: 'MIT' }` resolved via `spdx_id ?? key` to the empty string (?? does not skip blank), hit the `!resolved` branch, and returned NO_LICENSE_SENTINEL — discarding a valid license key. Switch all three normalizeLicense mirrors to trim-first + `||` so the first non-empty field (spdx_id preferred, else key) is selected: - src/utils/licenseFilter.ts - server/src/mcp/repoSearch.ts - electron/mcpLocalServer.js Regression: licenseFilter.test.ts covers blank/whitespace spdx_id → key fallback, both-blank → sentinel, and blank-spdx_id + 'Other' key → sentinel. Tests: vitest 143 passed. tsc clean for frontend and server. Co-Authored-By: Claude Opus 4.8 --- electron/mcpLocalServer.js | 4 ++-- server/src/mcp/repoSearch.ts | 6 +++--- src/utils/licenseFilter.test.ts | 13 +++++++++++++ src/utils/licenseFilter.ts | 10 ++++++---- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/electron/mcpLocalServer.js b/electron/mcpLocalServer.js index 8e32ac9ca..56d59b217 100644 --- a/electron/mcpLocalServer.js +++ b/electron/mcpLocalServer.js @@ -37,8 +37,8 @@ const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-licens function normalizeLicense(v) { if (v == null || v === '') return NO_LICENSE_SENTINEL; if (typeof v === 'object') { - const spdx = typeof v.spdx_id === 'string' ? v.spdx_id : null; - const key = typeof v.key === 'string' ? v.key : null; + const spdx = typeof v.spdx_id === 'string' ? v.spdx_id.trim() : ''; + const key = typeof v.key === 'string' ? v.key.trim() : ''; const resolved = spdx || key; if (!resolved) return NO_LICENSE_SENTINEL; return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; diff --git a/server/src/mcp/repoSearch.ts b/server/src/mcp/repoSearch.ts index 0b267e560..2977b1a72 100644 --- a/server/src/mcp/repoSearch.ts +++ b/server/src/mcp/repoSearch.ts @@ -13,9 +13,9 @@ export function normalizeLicense(v: unknown): string { if (v == null || v === '') return NO_LICENSE_SENTINEL; if (typeof v === 'object') { const l = v as { spdx_id?: unknown; key?: unknown }; - const spdx = typeof l.spdx_id === 'string' ? l.spdx_id : null; - const key = typeof l.key === 'string' ? l.key : null; - const resolved = spdx ?? key; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + const resolved = spdx || key; if (!resolved) return NO_LICENSE_SENTINEL; return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; } diff --git a/src/utils/licenseFilter.test.ts b/src/utils/licenseFilter.test.ts index 1ecac1054..051c9a0c0 100644 --- a/src/utils/licenseFilter.test.ts +++ b/src/utils/licenseFilter.test.ts @@ -68,6 +68,19 @@ describe('normalizeLicense', () => { // String already carrying the sentinel stays. expect(normalizeLicense(NO_LICENSE_SENTINEL)).toBe(NO_LICENSE_SENTINEL); }); + + it('falls back from blank/whitespace spdx_id to a valid key', () => { + // Regression: `spdx_id ?? key` retained an empty-string spdx_id and wrongly returned + // the no-license sentinel, discarding a valid key. Trim-first + `||` must select the + // first non-empty field (spdx_id preferred, else key). + expect(normalizeLicense({ spdx_id: '', key: 'MIT' } as unknown)).toBe('MIT'); + expect(normalizeLicense({ spdx_id: ' ', key: 'apache-2.0', name: 'Apache' } as unknown)) + .toBe('apache-2.0'); + // Both blank → sentinel. + expect(normalizeLicense({ spdx_id: '', key: ' ' } as unknown)).toBe(NO_LICENSE_SENTINEL); + // key 'Other' still collapses to sentinel once spdx_id is absent/blank. + expect(normalizeLicense({ spdx_id: '', key: 'Other' } as unknown)).toBe(NO_LICENSE_SENTINEL); + }); }); describe('buildEmbeddingText license', () => { diff --git a/src/utils/licenseFilter.ts b/src/utils/licenseFilter.ts index b71b868f3..e9a2f708e 100644 --- a/src/utils/licenseFilter.ts +++ b/src/utils/licenseFilter.ts @@ -37,11 +37,13 @@ const NOASSERTION_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-licens export function normalizeLicense(v: unknown): string { if (v == null || v === '') return NO_LICENSE_SENTINEL; if (typeof v === 'object') { - // GitHub license 对象：优先 spdx_id（如 'MIT'），回退 key（如 'Other'） + // GitHub license 对象：优先 spdx_id（如 'MIT'），回退 key（如 'Other'）。 + // 注意 trimmed-first + `||`：若 spdx_id 为空白串（≠ null），不可用 `??` 否则会保留空串 + // 并错误归入「无 license」，应回退到非空 key。 const l = v as { spdx_id?: unknown; key?: unknown }; - const spdx = typeof l.spdx_id === 'string' ? l.spdx_id : null; - const key = typeof l.key === 'string' ? l.key : null; - const resolved = spdx ?? key; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + const resolved = spdx || key; if (!resolved) return NO_LICENSE_SENTINEL; return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; } From cd4f176c5d2aa74c095421a324827629ad20b581 Mon Sep 17 00:00:00 2001 From: AmintaCCCP Date: Thu, 30 Jul 2026 19:02:17 +0800 Subject: [PATCH 6/7] fix: address remaining CodeRabbit audit findings on PR #254 - sync.ts: fix INSERT VALUES placeholder count (28 cols / 28 ?) - sync import: canonicalize license + vector_indexed_license to SPDX-or-null - toLicenseSpdxId: trim and accept only string spdx_id/key fields - embedding/rerank: normalizeLicense before writing License text - DataManagementPanel: default licenses to [] on legacy import - normalizeLicense: trim scalar strings before sentinel matching - tests: cover whitespace-padded sentinel variants --- electron/mcpLocalServer.js | 6 +++- server/src/mcp/repoSearch.ts | 6 +++- server/src/routes/repositories.ts | 9 +++-- server/src/routes/sync.ts | 36 ++++++++++++++----- .../settings/DataManagementPanel.tsx | 9 ++++- src/services/aiService.ts | 8 +++-- src/services/githubApi.ts | 9 +++-- src/services/vectorSearchService.ts | 10 ++++-- src/utils/licenseFilter.test.ts | 9 +++++ src/utils/licenseFilter.ts | 6 +++- 10 files changed, 84 insertions(+), 24 deletions(-) diff --git a/electron/mcpLocalServer.js b/electron/mcpLocalServer.js index 56d59b217..5d48690cb 100644 --- a/electron/mcpLocalServer.js +++ b/electron/mcpLocalServer.js @@ -44,7 +44,11 @@ function normalizeLicense(v) { return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; } if (typeof v !== 'string') return NO_LICENSE_SENTINEL; - return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; + // 直接字符串路径也需 trim：避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并 + const normalized = v.trim(); + return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase()) + ? NO_LICENSE_SENTINEL + : normalized; } function projectRepo(repo, max = 400) { diff --git a/server/src/mcp/repoSearch.ts b/server/src/mcp/repoSearch.ts index 2977b1a72..cf889ca6f 100644 --- a/server/src/mcp/repoSearch.ts +++ b/server/src/mcp/repoSearch.ts @@ -20,7 +20,11 @@ export function normalizeLicense(v: unknown): string { return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; } if (typeof v !== 'string') return NO_LICENSE_SENTINEL; - return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; + // 直接字符串路径也需 trim：避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并 + const normalized = v.trim(); + return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase()) + ? NO_LICENSE_SENTINEL + : normalized; } export interface McpRepository { diff --git a/server/src/routes/repositories.ts b/server/src/routes/repositories.ts index 13b3008b5..93eb669e0 100644 --- a/server/src/routes/repositories.ts +++ b/server/src/routes/repositories.ts @@ -19,10 +19,13 @@ function parseJsonColumn(value: unknown): unknown[] { */ function toLicenseSpdxId(license: unknown): string | null { if (license == null) return null; - if (typeof license === 'string') return license || null; + if (typeof license === 'string') return license.trim() || null; if (typeof license === 'object') { - const l = license as { spdx_id?: string; key?: string }; - return l.spdx_id || l.key || null; + // 运行时校验：malformed 备份/第三方源可能把 spdx_id/key 写成非字符串 + const l = license as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + return spdx || key || null; } return null; } diff --git a/server/src/routes/sync.ts b/server/src/routes/sync.ts index f64fd5451..e4f5b4086 100644 --- a/server/src/routes/sync.ts +++ b/server/src/routes/sync.ts @@ -5,6 +5,19 @@ import { config } from '../config.js'; const router = Router(); +/** 与前端 NOASSERTION_KEYS 对齐：空白 / NOASSERTION / Other / none 等落 null。 */ +const NO_LICENSE_KEYS = new Set(['', 'noassertion', 'other', 'none', 'no-license']); + +/** + * 把导入的 license 字符串规范为 SPDX id 或 null。 + * trim 后若为空或落入「无 license」集合则返回 null，否则返回 trim 后的原值。 + */ +function canonicalizeLicenseString(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed || NO_LICENSE_KEYS.has(trimmed.toLowerCase())) return null; + return trimmed; +} + function maskApiKey(key: string | null | undefined): string { if (!key || typeof key !== 'string') return ''; if (key.length <= 4) return '****'; @@ -109,7 +122,7 @@ router.post('/api/sync/import', (req, res) => { ai_summary, ai_tags, ai_platforms, analyzed_at, analysis_failed, custom_description, custom_tags, custom_category, category_locked, last_edited, subscribed_to_releases, vector_indexed_at, license, vector_indexed_license - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); for (const r of repos) { // 验证必需的字段 @@ -117,18 +130,24 @@ router.post('/api/sync/import', (req, res) => { throw new Error(`Invalid repository data: missing or invalid id`); } // license：兼容旧备份（无该列→null）、GitHub 对象形态、已规范化的 SPDX 字符串。 - // 仅当候选字段在运行时确为字符串时才采用，避免 malformed 值（如 spdx_id: 123） - // 使 licenseValue 变成非字符串、污染导入事务；否则归一化为 SPDX id 或 null。 + // 候选字符串 trim 后，空白 / NOASSERTION / Other / none 统一落 null（与 + // 前端 normalizeLicense 的「无 license」语义对齐），保证 DB 只存 SPDX-or-null。 const rawLicense = (r as Record).license; let licenseValue: string | null = null; if (typeof rawLicense === 'string') { - licenseValue = rawLicense || null; + licenseValue = canonicalizeLicenseString(rawLicense); } else if (rawLicense && typeof rawLicense === 'object') { const obj = rawLicense as { spdx_id?: unknown; key?: unknown }; - const spdx = typeof obj.spdx_id === 'string' ? obj.spdx_id : null; - const key = typeof obj.key === 'string' ? obj.key : null; - licenseValue = spdx || key || null; + const spdx = typeof obj.spdx_id === 'string' ? obj.spdx_id.trim() : ''; + const key = typeof obj.key === 'string' ? obj.key.trim() : ''; + licenseValue = canonicalizeLicenseString(spdx || key); } + // vector_indexed_license 与 license 同一套 SPDX-or-null 规则，避免指纹与当前 + // license 语义分裂（例如 "NOASSERTION" 原样入库后增量谓词永远判定为变更）。 + const rawVectorLicense = (r as Record).vector_indexed_license; + const vectorIndexedLicense = typeof rawVectorLicense === 'string' + ? canonicalizeLicenseString(rawVectorLicense) + : null; repoStmt.run( r.id, r.name, r.full_name, r.description ?? null, r.html_url, r.stargazers_count ?? 0, r.language ?? null, @@ -146,10 +165,9 @@ router.post('/api/sync/import', (req, res) => { r.subscribed_to_releases ? 1 : 0, r.vector_indexed_at ?? null, licenseValue, - // vector_indexed_license：仅接受字符串（已是规范化 SPDX id），否则 null。 // INSERT OR REPLACE 会整行替换，故备份无此列时落 null（影响：增量谓词会 // 触发一次重索引回填指纹），合预期。 - typeof r.vector_indexed_license === 'string' ? r.vector_indexed_license || null : null + vectorIndexedLicense ); } counts.repositories = repos.length; diff --git a/src/components/settings/DataManagementPanel.tsx b/src/components/settings/DataManagementPanel.tsx index d74780472..e3fdb9a27 100644 --- a/src/components/settings/DataManagementPanel.tsx +++ b/src/components/settings/DataManagementPanel.tsx @@ -670,7 +670,14 @@ export const DataManagementPanel: React.FC = ({ t }) = useAppStore.setState({ readReleases: new Set(importedData.readReleases || []) }); } if (selectedTypes.includes('searchFilters') && importedData.searchFilters) { - useAppStore.setState({ searchFilters: importedData.searchFilters }); + // 旧备份可能缺少新增的 licenses 字段，导入时默认 [] 保持 SearchFilters 契约 + const importedFilters = importedData.searchFilters; + useAppStore.setState({ + searchFilters: { + ...importedFilters, + licenses: importedFilters.licenses ?? [], + }, + }); } if (selectedTypes.includes('uiSettings')) { if (importedData.theme === 'light' || importedData.theme === 'dark') { diff --git a/src/services/aiService.ts b/src/services/aiService.ts index 73e466086..0b68e98f6 100644 --- a/src/services/aiService.ts +++ b/src/services/aiService.ts @@ -1,7 +1,7 @@ import { Repository, Gist, AIConfig, AIApiType } from '../types'; import { backend } from './backendAdapter'; import { buildApiUrl, buildFinalApiUrl } from '../utils/apiUrlBuilder'; -import { normalizeLicense } from '../utils/licenseFilter'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; import { logger } from './logger'; interface OpenAIResponseContentPart { @@ -739,7 +739,11 @@ AI Summary: ${gist.ai_summary || 'None'}`; const parts = [`${index + 1}. ID: ${repo.id} | ${repo.full_name}`]; if (desc) parts.push(` ${desc}`); const meta = [repo.language, `★${stars}`]; - if (repo.license) meta.push(`License: ${repo.license}`); + // 与 embedding 一致：归一化后、非哨兵才写入，避免 raw 对象变成 "[object Object]" + { + const lic = normalizeLicense(repo.license); + if (lic !== NO_LICENSE_SENTINEL) meta.push(`License: ${lic}`); + } if (tags) meta.push(`Tags: ${tags}`); parts.push(` ${meta.join(' | ')}`); return parts.join('\n'); diff --git a/src/services/githubApi.ts b/src/services/githubApi.ts index c109956dd..58978251e 100644 --- a/src/services/githubApi.ts +++ b/src/services/githubApi.ts @@ -47,10 +47,13 @@ interface GitHubStarredItem { */ function toLicenseSpdxId(license: unknown): string | null { if (license == null) return null; - if (typeof license === 'string') return license || null; + if (typeof license === 'string') return license.trim() || null; if (typeof license === 'object') { - const l = license as { spdx_id?: string; key?: string }; - return l.spdx_id || l.key || null; + // 运行时校验：malformed 备份/第三方源可能把 spdx_id/key 写成非字符串 + const l = license as { spdx_id?: unknown; key?: unknown }; + const spdx = typeof l.spdx_id === 'string' ? l.spdx_id.trim() : ''; + const key = typeof l.key === 'string' ? l.key.trim() : ''; + return spdx || key || null; } return null; } diff --git a/src/services/vectorSearchService.ts b/src/services/vectorSearchService.ts index ed3907d74..37786d577 100644 --- a/src/services/vectorSearchService.ts +++ b/src/services/vectorSearchService.ts @@ -6,7 +6,7 @@ */ import type { EmbeddingConfig, VectorSearchConfig, Repository } from '../types'; -import { normalizeLicense } from '../utils/licenseFilter'; +import { NO_LICENSE_SENTINEL, normalizeLicense } from '../utils/licenseFilter'; // ============================================================ // EmbeddingClient @@ -363,8 +363,12 @@ export function buildEmbeddingText(repo: Repository, readmeContent?: string, max if (allTopics.length > 0) parts.push(`Topics: ${allTopics.join(', ')}`); if (repo.language) parts.push(`Language: ${repo.language}`); - // 开源许可：SPDX id（如 MIT），短文本，对 embedding token 成本可忽略 - if (repo.license) parts.push(`License: ${repo.license}`); + // 开源许可：先归一化，避免 raw GitHub 对象变成 "[object Object]" 污染 embedding； + // 哨兵（无/未声明）不写入，与 null/缺失保持同一语义。 + { + const lic = normalizeLicense(repo.license); + if (lic !== NO_LICENSE_SENTINEL) parts.push(`License: ${lic}`); + } // README 内容提供最丰富的语义信息 if (readmeContent) { diff --git a/src/utils/licenseFilter.test.ts b/src/utils/licenseFilter.test.ts index 051c9a0c0..7842243c1 100644 --- a/src/utils/licenseFilter.test.ts +++ b/src/utils/licenseFilter.test.ts @@ -47,6 +47,15 @@ describe('normalizeLicense', () => { expect(normalizeLicense('none')).toBe(NO_LICENSE_SENTINEL); }); + it('trims scalar strings before sentinel matching and SPDX pass-through', () => { + // 直接字符串路径此前未 trim，" Other " 会成为独立 facet，拆分过滤/统计。 + expect(normalizeLicense(' Other ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' NOASSERTION ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' none ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' ')).toBe(NO_LICENSE_SENTINEL); + expect(normalizeLicense(' MIT ')).toBe('MIT'); + }); + it('never throws on a non-string license (GitHub object / number / legacy data)', () => { // Regression: performBasicTextSearch / performEnhancedBasicSearch / RepositoryCard call // normalizeLicense(repo.license) during render. A repo that never passed through diff --git a/src/utils/licenseFilter.ts b/src/utils/licenseFilter.ts index e9a2f708e..fddacf427 100644 --- a/src/utils/licenseFilter.ts +++ b/src/utils/licenseFilter.ts @@ -48,5 +48,9 @@ export function normalizeLicense(v: unknown): string { return NOASSERTION_KEYS.has(resolved.toLowerCase()) ? NO_LICENSE_SENTINEL : resolved; } if (typeof v !== 'string') return NO_LICENSE_SENTINEL; // 数字等为非合法 license - return NOASSERTION_KEYS.has(v.toLowerCase()) ? NO_LICENSE_SENTINEL : v; + // 直接字符串路径也需 trim：避免 " Other " / " NOASSERTION " 等空白变体逃过哨兵归并 + const normalized = v.trim(); + return !normalized || NOASSERTION_KEYS.has(normalized.toLowerCase()) + ? NO_LICENSE_SENTINEL + : normalized; } From 173677e5557668f86645a1436ae46e75808b27a0 Mon Sep 17 00:00:00 2001 From: AmintaCCCP Date: Thu, 30 Jul 2026 19:25:54 +0800 Subject: [PATCH 7/7] fix: shrink repository card stats row to stop license overflow Reduce the language/stars/license meta row from text-sm (14px) to text-xs (12px), shrink Star/Scale icons to w-3.5/h-3.5, color dot to w-2.5/h-2.5, and add min-w-0/flex-shrink-0 + truncate caps so the license SPDX id no longer overflows the card container when present. --- src/components/RepositoryCard.tsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/components/RepositoryCard.tsx b/src/components/RepositoryCard.tsx index 0bcd3d825..74c25f1af 100644 --- a/src/components/RepositoryCard.tsx +++ b/src/components/RepositoryCard.tsx @@ -1123,27 +1123,27 @@ const RepositoryCardComponent: React.FC = ({ {/* Stats */}
{/* Language and Stars */} -
+
{repository.language && ( -
+
{repository.language}
)} -
- - {formatNumber(repository.stargazers_count)} +
+ + {formatNumber(repository.stargazers_count)}
{(() => { // license：归一化后展示 SPDX id；无 license 不渲染 const lic = normalizeLicense(repository.license); if (lic === NO_LICENSE_SENTINEL) return null; return ( -
- +
+ {lic}
);