diff --git a/cloudflare-worker/src/index.ts b/cloudflare-worker/src/index.ts index 043edf9e3..419901ba2 100644 --- a/cloudflare-worker/src/index.ts +++ b/cloudflare-worker/src/index.ts @@ -1,3 +1,5 @@ +import { compactUpsertVectors } from './metadata'; + /** * GitHub Stars Vectorize — 极简代理 Worker * @@ -24,6 +26,11 @@ interface UpsertRequest { vectors: VectorizeVector[]; } +// returnMetadata:'all' caps topK at 50 on Vectorize V2 indexes; clamp requests +// to avoid passing an out-of-range topK and being silently truncated. +const QUERY_TOPK_LIMIT = 50; +const QUERY_DEFAULT_TOPK = 20; + const CORS_HEADERS: Record = { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', @@ -62,18 +69,23 @@ export default { if (!Array.isArray(vectors) || vectors.length === 0) { return jsonResponse({ success: false, error: 'vectors array required' }, 400); } - await env.VECTORIZE.upsert(vectors); + const compactedVectors = compactUpsertVectors(vectors); + await env.VECTORIZE.upsert(compactedVectors); return jsonResponse({ success: true, upserted: vectors.length }); } // POST /query — 向量相似度查询 if (request.method === 'POST' && url.pathname === '/query') { - const { vector, topK = 20, threshold = 0.35 } = (await request.json()) as QueryRequest; + const { vector, topK = QUERY_DEFAULT_TOPK, threshold = 0.35 } = (await request.json()) as QueryRequest; if (!Array.isArray(vector) || vector.length === 0) { return jsonResponse({ success: false, error: 'vector array required' }, 400); } - // returnMetadata:'all' caps topK at 50; clamp to avoid silent truncation - const clampedTopK = Math.min(topK, 50); + // returnMetadata:'all' caps topK at 50 (Vectorize V2 limits); clamp so + // an oversized/negative/NaN topK degrades gracefully instead of erroring. + const requestedTopK = typeof topK === 'number' && Number.isFinite(topK) + ? Math.floor(topK) + : QUERY_DEFAULT_TOPK; + const clampedTopK = Math.min(Math.max(requestedTopK, 1), QUERY_TOPK_LIMIT); const matches = await env.VECTORIZE.query(vector, { topK: clampedTopK, returnMetadata: 'all' as const, diff --git a/cloudflare-worker/src/metadata.ts b/cloudflare-worker/src/metadata.ts new file mode 100644 index 000000000..2140796ed --- /dev/null +++ b/cloudflare-worker/src/metadata.ts @@ -0,0 +1,197 @@ +/** + * Vectorize metadata compaction helpers. + * + * Kept intentionally free of `@cloudflare/workers-types` so that both the Worker + * (which sees those globals) and the app's vitest suite (type-checked against + * tsconfig.app.json) can import it without dragging workers globals into the + * app's TS program. + */ + +export type CompactMetadata = { + full_name: string; + description: string; + language: string; + stars: number; + tags: string[]; + license?: string; +}; + +/** Mirror of Vectorize's metadata value types, kept free of workers-types. */ +export type VectorMetadataValue = string | number | boolean | string[]; +export type VectorMetadata = VectorMetadataValue | Record; + +/** Vectorize's hard per-vector metadata limit (10 KiB). */ +export const VECTORIZE_METADATA_LIMIT_BYTES = 10_240; + +/** + * Keep a margin below Vectorize's 10 KiB per-vector metadata limit so that + * future small fields or JSON escape overhead do not turn a valid payload into + * a 40016. + */ +export const VECTORIZE_METADATA_SAFE_BYTES = 9_500; + +const encoder = new TextEncoder(); + +export function utf8ByteLength(value: string): number { + return encoder.encode(value).byteLength; +} + +export function jsonByteLength(value: unknown): number { + return utf8ByteLength(JSON.stringify(value) ?? ''); +} + +/** Truncate on Unicode code-point boundaries, using UTF-8 bytes rather than JS characters. */ +export function truncateUtf8(value: string, maxBytes: number): string { + if (maxBytes <= 0) return ''; + if (utf8ByteLength(value) <= maxBytes) return value; + + const codePoints = Array.from(value); + let low = 0; + let high = codePoints.length; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (utf8ByteLength(codePoints.slice(0, middle).join('')) <= maxBytes) { + low = middle; + } else { + high = middle - 1; + } + } + return codePoints.slice(0, low).join(''); +} + +/** Coerce an arbitrary input into the well-typed six-field schema. Only used as a fallback. */ +function normalizeMetadata(input: unknown): CompactMetadata { + const source = input && typeof input === 'object' && !Array.isArray(input) + ? input as Record + : {}; + const metadata: CompactMetadata = { + full_name: typeof source.full_name === 'string' ? source.full_name : '', + description: typeof source.description === 'string' ? source.description : '', + language: typeof source.language === 'string' ? source.language : '', + stars: typeof source.stars === 'number' && Number.isFinite(source.stars) ? source.stars : 0, + tags: Array.isArray(source.tags) + ? source.tags.filter((tag): tag is string => typeof tag === 'string') + : [], + }; + if (typeof source.license === 'string') metadata.license = source.license; + return metadata; +} + +function truncateTags(tags: string[], maxBytes: number): string[] { + if (maxBytes <= 0) return []; + const result: string[] = []; + for (const tag of tags) { + const candidate = [...result, tag]; + if (jsonByteLength(candidate) <= maxBytes) { + result.push(tag); + continue; + } + + // Keep a prefix of the first tag that does not fit, then stop: later tags + // cannot be more useful than the already retained prefix under this budget. + const codePoints = Array.from(tag); + let low = 0; + let high = codePoints.length; + let best = ''; + while (low <= high) { + const middle = Math.floor((low + high) / 2); + const shortened = codePoints.slice(0, middle).join(''); + if (jsonByteLength([...result, shortened]) <= maxBytes) { + best = shortened; + low = middle + 1; + } else { + high = middle - 1; + } + } + if (best) result.push(best); + break; + } + return result; +} + +/** + * Compact one vector's metadata independently of the rest of its batch. + * + * Priority, from least to most destructive: + * 1. Metadata already within the budget is returned untouched, preserving every + * field — including any outside the known schema. + * 2. Over budget: the variable `description` / `tags` fields are reduced first + * (largest field first, then both together) while every other field, known or + * unknown, is preserved. + * 3. Still over budget: fall back to the six-field schema with truncated fixed + * fields, protecting against malformed input such as an enormous license or + * repository name. + */ +export function compactVectorMetadata(input: unknown): Record { + const source = input && typeof input === 'object' && !Array.isArray(input) + ? input as Record + : null; + if (source && jsonByteLength(source) <= VECTORIZE_METADATA_SAFE_BYTES) { + return { ...source } as Record; + } + + const metadata = normalizeMetadata(input); + if (jsonByteLength(metadata) <= VECTORIZE_METADATA_SAFE_BYTES) { + return { ...metadata } as Record; + } + + const variableBytes = Math.max( + utf8ByteLength(metadata.description), + jsonByteLength(metadata.tags), + ); + let low = 0; + let high = variableBytes; + let best = { ...metadata, description: '', tags: [] } as Record; + + // A shared cap implements the "largest field first, then both together" + // policy: a smaller field is untouched until the larger one reaches it. + while (low <= high) { + const cap = Math.floor((low + high) / 2); + const candidate = { + ...(source ?? metadata), + description: truncateUtf8(metadata.description, cap), + tags: truncateTags(metadata.tags, cap), + } as Record; + if (jsonByteLength(candidate) <= VECTORIZE_METADATA_SAFE_BYTES) { + best = candidate; + low = cap + 1; + } else { + high = cap - 1; + } + } + + if (jsonByteLength(best) <= VECTORIZE_METADATA_SAFE_BYTES) return best; + + // The known fields are already small; this final fallback protects against + // malformed input such as an enormous license, repository name, or unknown field. + const fallback: CompactMetadata = { + full_name: truncateUtf8(metadata.full_name, 512), + description: '', + language: truncateUtf8(metadata.language, 128), + stars: metadata.stars, + tags: [], + }; + if (metadata.license) fallback.license = truncateUtf8(metadata.license, 256); + if (jsonByteLength(fallback) > VECTORIZE_METADATA_SAFE_BYTES) delete fallback.license; + return fallback as Record; +} + +/** + * Apply per-vector metadata compaction at the /upsert boundary. + * Preserves every top-level vector property (id, values, namespace, …) while + * replacing `metadata` with its compacted form. Vectors that omit `metadata` + * keep it omitted — no empty schema object is fabricated on their behalf. + */ +export function compactUpsertVectors( + vectors: readonly T[], +): Array }> { + return vectors.map((vector) => { + if (vector.metadata === undefined) { + return { ...vector }; + } + return { + ...vector, + metadata: compactVectorMetadata(vector.metadata), + }; + }) as Array }>; +} \ No newline at end of file diff --git a/src/services/cloudflareWorkerMetadata.test.ts b/src/services/cloudflareWorkerMetadata.test.ts new file mode 100644 index 000000000..1eeb9863e --- /dev/null +++ b/src/services/cloudflareWorkerMetadata.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from 'vitest'; +import { + compactUpsertVectors, + compactVectorMetadata, + VECTORIZE_METADATA_LIMIT_BYTES, +} from '../../cloudflare-worker/src/metadata'; + +const jsonBytes = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)).byteLength; + +const baseMetadata = { + full_name: 'owner/repository', + description: 'A short description', + language: 'TypeScript', + stars: 42, + tags: ['vector-search', 'cloudflare'], + license: 'MIT', +}; + +type TestMetadata = { + full_name: string; + description: string; + language: string; + stars: number; + tags: string[]; + license?: string; + [key: string]: unknown; +}; + +const compact = (input: unknown): TestMetadata => compactVectorMetadata(input) as TestMetadata; + +describe('Cloudflare Worker Vectorize metadata compaction', () => { + it('leaves metadata unchanged when it is within the budget', () => { + expect(compact(baseMetadata)).toEqual(baseMetadata); + }); + + it('truncates a long UTF-8 description while retaining small tags', () => { + const metadata = compact({ + ...baseMetadata, + description: '中文描述'.repeat(7_000), + tags: ['semantic-search'], + }); + + expect(jsonBytes(metadata)).toBeLessThanOrEqual(VECTORIZE_METADATA_LIMIT_BYTES); + expect(metadata.description).not.toBe('中文描述'.repeat(7_000)); + expect(metadata.description).toContain('中文描述'); + expect(metadata.tags).toEqual(['semantic-search']); + expect(metadata.description).not.toContain('\uFFFD'); + }); + + it('compresses both description and tags when both fields are oversized', () => { + const description = 'd'.repeat(20_000); + const tag = '标签'.repeat(8_000); + const metadata = compact({ + ...baseMetadata, + description, + tags: [tag], + }); + + expect(jsonBytes(metadata)).toBeLessThanOrEqual(VECTORIZE_METADATA_LIMIT_BYTES); + expect(metadata.description.length).toBeLessThan(description.length); + expect(metadata.tags[0]?.length ?? 0).toBeLessThan(tag.length); + expect(metadata.full_name).toBe(baseMetadata.full_name); + expect(metadata.language).toBe(baseMetadata.language); + expect(metadata.stars).toBe(baseMetadata.stars); + }); + + it('protects the limit from a malformed oversized license field', () => { + const metadata = compact({ + ...baseMetadata, + description: '', + tags: [], + license: 'license'.repeat(20_000), + }); + + expect(jsonBytes(metadata)).toBeLessThanOrEqual(VECTORIZE_METADATA_LIMIT_BYTES); + expect(metadata.full_name).toBe(baseMetadata.full_name); + expect(metadata.language).toBe(baseMetadata.language); + expect(metadata.stars).toBe(baseMetadata.stars); + }); + + it('preserves unknown metadata fields when the payload is within budget', () => { + const input = { + ...baseMetadata, + url: 'https://github.com/owner/repository', + source_id: 'abc-123', + custom: 123, + }; + + expect(compact(input)).toEqual(input); + }); + + it('reduces oversized description first while keeping unknown fields', () => { + const metadata = compact({ + ...baseMetadata, + description: 'x'.repeat(20_000), + url: 'https://github.com/owner/repository', + }); + + expect(jsonBytes(metadata)).toBeLessThanOrEqual(VECTORIZE_METADATA_LIMIT_BYTES); + expect(metadata.description.length).toBeLessThan(20_000); + expect(metadata.url).toBe('https://github.com/owner/repository'); + }); + + it('drops unknown fields only in the final fallback when they cannot fit', () => { + const metadata = compact({ + full_name: baseMetadata.full_name, + description: '', + language: baseMetadata.language, + stars: baseMetadata.stars, + tags: [], + license: baseMetadata.license, + url: 'u'.repeat(100_000), + }); + + expect(jsonBytes(metadata)).toBeLessThanOrEqual(VECTORIZE_METADATA_LIMIT_BYTES); + expect(metadata.full_name).toBe(baseMetadata.full_name); + expect(metadata).not.toHaveProperty('url'); + }); + + it('applies compaction to each vector at the upsert boundary', () => { + const vectors = [ + { id: '1', values: [0.1, 0.2], namespace: 'a', metadata: baseMetadata }, + { + id: '2', + values: [0.3], + namespace: 'b', + metadata: { ...baseMetadata, description: 'x'.repeat(20_000), url: 'https://example.com' }, + }, + ]; + + const compacted = compactUpsertVectors(vectors); + + // vector identity / top-level fields are preserved untouched + expect(compacted[0].id).toBe('1'); + expect(compacted[0].values).toEqual([0.1, 0.2]); + expect(compacted[0].namespace).toBe('a'); + // within-budget metadata is passed through byte-for-byte + expect(compacted[0].metadata).toEqual(baseMetadata); + + // oversized metadata is compacted but keeps unknown fields + expect(jsonBytes(compacted[1].metadata)).toBeLessThanOrEqual(VECTORIZE_METADATA_LIMIT_BYTES); + expect(compacted[1].metadata.url).toBe('https://example.com'); + expect((compacted[1].metadata.description as string).length).toBeLessThan(20_000); + expect(compacted[1].namespace).toBe('b'); + }); + + it('keeps metadata omitted when a vector has no metadata property', () => { + const compacted = compactUpsertVectors([ + { id: '3', values: [0.9], namespace: 'c' }, + { id: '4', values: [0.8], namespace: 'd', metadata: undefined }, + ]); + + expect(compacted[0]).not.toHaveProperty('metadata'); + expect(compacted[0].id).toBe('3'); + expect(compacted[0].namespace).toBe('c'); + expect(compacted[1].metadata).toBeUndefined(); + expect(compacted[1].id).toBe('4'); + }); +}); diff --git a/src/services/vectorSearchService.ts b/src/services/vectorSearchService.ts index 581fa102d..a42633794 100644 --- a/src/services/vectorSearchService.ts +++ b/src/services/vectorSearchService.ts @@ -179,6 +179,10 @@ export class EmbeddingClient { // VectorSearchService — 与 Cloudflare Worker 通信 // ============================================================ +// Vectorize V2 限制:returnMetadata 返回 metadata 时 topK 上限为 50。 +// Worker /query 同样以 50 为 clamp 上限,此处保持一致,避免请求被静默截断。 +export const VECTORIZE_QUERY_TOPK_LIMIT = 50; + export interface VectorizeVector { id: string; values: number[]; @@ -700,9 +704,10 @@ export async function findSimilarRepositories( const queryVector = vectors[0]; // 2. 向量检索(多取 1 个以容纳源仓库自身,随后过滤) + // 请求的 topK 需要带上源仓库的 +1 余量,但不超过 Vectorize 的 50 上限。 const matches = await vectorService.query( queryVector, - { topK: topK + 1, threshold }, + { topK: Math.min(topK + 1, VECTORIZE_QUERY_TOPK_LIMIT), threshold }, signal );