diff --git a/AGENTS.md b/AGENTS.md index 6aabfdc7..087bd5b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -> Current objective: **retrieval accuracy through default hybrid + hosted Jina**. +> Current objective: **retrieval accuracy through default hybrid + hosted Gemini 001**. > Do not trade retrieval breadth or relevance for a speed-only improvement. > The smaller ANN candidate pool was discarded; accuracy is the experiment goal. @@ -49,10 +49,10 @@ python3 scripts/e2e_mcp_test.py \ - `src/snapshot.zig` — sensitive file filtering - `install/install.sh` — binary download and config modification -## Jina hybrid accuracy experiments +## Prior Jina hybrid accuracy experiments -Optimize accuracy through default hybrid, hosted Jina and local OpenPuffer mmap. -Preserve calibration, hosted exact fallback, and the 4× ANN pool. Do not run local +The Jina results below are historical baselines; the managed default is now +Gemini 001. Preserve calibration, hosted exact fallback, and the 4× ANN pool. Do not run local models or exchange retrieval breadth for speed. Release integration: 0.2.5853, based on the validated 0.2.5852 experiments. Old main Qwen experiments are obsolete and do not govern this integration. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b5c93f9..4c7d2ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 0.2.5858 - 2026-09-27 + +- Hosted semantic retrieval now uses `gemini-embedding-001` (512D) by default. Queries carry a distinct Gemini query identity. Existing sidecars built with the former hosted Jina or Qwen defaults migrate once in the background; custom endpoints and model overrides are unchanged. Set `CODEDB_EMBEDDINGS_MODEL=jinaai/jina-embeddings-v2-base-code` to keep Jina explicitly. + ## 0.2.5857 - 2026-09-26 - Auto-register codedb in additional detected MCP clients: oh-my-pi, Hermes, Qwen Code, ZCode, Trae, Cline, Copilot CLI, Antigravity, Kiro, and OpenCode. `codedb nuke` removes the matching entries. Invalid JSON is left untouched. diff --git a/README.md b/README.md index ee55d495..0f4eac9b 100644 --- a/README.md +++ b/README.md @@ -476,19 +476,17 @@ Build a new local ANN explicitly with: codedb /path/to/repo semantic-index ``` -CodeDB 0.2.5851 staged the managed embedding migration without breaking older -clients. Versions through 0.2.5850 continue to request -`Qwen/Qwen3-Embedding-0.6B`; 0.2.5851 and later request -`jinaai/jina-embeddings-v2-base-code`. The service keeps both routes live. An -existing Qwen ANN sidecar is rejected by the new client's model/vector-space -check. Starting in 0.2.5852, the first hybrid MCP query still returns through -the bounded exact Jina reranker, then schedules one background rebuild of that -exact legacy hosted-Qwen sidecar. The old OpenPuffer generation remains intact -until the Jina replacement passes source, Git, metadata, and vector-space -validation and commits atomically. Custom endpoints/model overrides and new -repositories are never auto-built; set `CODEDB_NO_AUTO_SEMANTIC_MIGRATION=1` -to disable even this legacy migration. CodeDB never mixes vectors from the two -models. +The managed default is now `gemini-embedding-001` at 512 dimensions. Clients +through 0.2.5850 requested Qwen 0.6B; versions 0.2.5851–0.2.5856 requested +Jina v2 code. The hosted service keeps those identifiers separate for existing +clients. Old Qwen and Jina ANN sidecars fail the new model/vector-space check. +For an exact former hosted-default sidecar, the first hybrid query uses bounded +exact Gemini reranking and schedules a single background rebuild. The old +OpenPuffer generation stays intact until its Gemini replacement passes source, +Git, metadata, and vector-space validation and commits atomically. Custom +endpoints/model overrides and new repositories are never auto-built; set +`CODEDB_NO_AUTO_SEMANTIC_MIGRATION=1` to disable even this migration. CodeDB +never mixes vectors from different models. It splits already-indexable files into bounded 832-byte source chunks, uses four concurrent 25-item requests by default (explicitly configurable from one @@ -510,8 +508,10 @@ enrolls its public key, and receives a short-lived server-signed certificate. Every request proves possession of the installation key; renewal is automatic. The private seed stays in a global `~/.codedb/credentials.json` file written atomically with mode `0600` on POSIX and is never part of an embedding request. -The public route exposes only the managed CodeDB lane and enforces enrollment, -installation, and aggregate network limits. General provider APIs remain authenticated. +The public route exposes only the managed CodeDB lane, forwards bounded text +to Gemini 001 under CodeDB's zero-retention service policy, and enforces +enrollment, installation, and aggregate network limits. General provider APIs +remain authenticated. `format=json` exposes this boundary in the `retrieval` object, including vector dimensions, bounded byte/document counts, retention policy, and failure diff --git a/build.zig.zon b/build.zig.zon index 65cf2f0c..ac9b559f 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ .name = .codedb2, .fingerprint = 0x6e18d96ca2a31757, - .version = "0.2.5857", + .version = "0.2.5858", .minimum_zig_version = "0.17.0-dev.813+2153f8143", .dependencies = .{ .nanoregex = .{ diff --git a/docs/cli.md b/docs/cli.md index 5bc4eb37..ffd0f515 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -116,23 +116,22 @@ paths cannot enter a batch even from a stale index. ### Managed model migration -CodeDB 0.2.5851 changes the managed default to -`jinaai/jina-embeddings-v2-base-code`; clients through 0.2.5850 continue to -request `Qwen/Qwen3-Embedding-0.6B`, and the hosted service serves both model -names during the migration. Sidecar metadata includes the model, dimensions, -query encoding, document-card encoding, and a calibration vector. A new client -therefore refuses an old Qwen sidecar rather than searching it with Jina -vectors. In 0.2.5852, the first hybrid MCP query returns via transient exact -reranking and schedules one background rebuild when—and only when—the sidecar -exactly matches CodeDB's former hosted-Qwen vector space. You can still replace -it explicitly: +The managed default is `gemini-embedding-001` at 512 dimensions. Clients +through 0.2.5850 used Qwen 0.6B; versions 0.2.5851–0.2.5856 used Jina v2 +code. The hosted service serves distinct model identifiers during migration. +Sidecar metadata includes the model, dimensions, query encoding, document-card +encoding, and a calibration vector. A new client rejects an old Qwen or Jina +sidecar rather than mixing its vectors with Gemini. The first hybrid query +returns via transient exact Gemini reranking and schedules one background rebuild +only when the sidecar exactly matches a former hosted-default vector space. +You can still replace it explicitly: ```bash codedb /path/to/repo semantic-index ``` Both paths replace the old generation transactionally. The previous sidecar is -kept until the new Jina generation has passed source/Git freshness and metadata +kept until the new Gemini generation has passed source/Git freshness and metadata validation. Custom endpoints, model overrides, fresh repositories, Windows (until OpenPuffer has native mmap loading there), and `CODEDB_NO_AUTO_SEMANTIC_MIGRATION=1` remain explicit-only. @@ -158,7 +157,7 @@ codedb-cli stop # stop daemon | `CODEDB_PORT` | `7719` | HTTP port for the daemon | | `CODEDB_BINARY` | `codedb` | Path to the codedb binary | | `CODEDB_EMBEDDINGS_URL` | `https://embeddings.wiki.codes/v1/codedb/embeddings` | Free bounded hosted lane; may be overridden with another HTTPS endpoint | -| `CODEDB_EMBEDDINGS_MODEL` | managed | Provider deployment/model identifier; set only with a custom endpoint | +| `CODEDB_EMBEDDINGS_MODEL` | `gemini-embedding-001` | Managed model identifier; override only when deliberately using another vector space | | `CODEDB_EMBEDDINGS_DIMENSIONS` | managed | Requested custom-provider dimensions (64-4096) | | `CODEDB_EMBEDDINGS_TOKEN` | unset | Optional legacy bearer token for protected/custom endpoints; the hosted lane enrolls automatically | | `CODEDB_EMBEDDINGS_TIMEOUT_MS` | `15000` | Per-request deadline in milliseconds (10-120000) | diff --git a/docs/releases/0.2.5858.md b/docs/releases/0.2.5858.md new file mode 100644 index 00000000..55ff5a22 --- /dev/null +++ b/docs/releases/0.2.5858.md @@ -0,0 +1,18 @@ +# CodeDB 0.2.5858 + +Hosted semantic retrieval moves to Gemini `gemini-embedding-001`. + +## Gemini hosted embeddings + +0.2.5857 and earlier requested `jinaai/jina-embeddings-v2-base-code` from the hosted embeddings endpoint. The default is now `gemini-embedding-001` at the same 512D contract, with its own query encoding and vector-space identity so Gemini vectors are never mixed with older ones. + +A local ANN sidecar built with the former hosted Jina or Qwen default is rebuilt once in the background the first time a hybrid query sees it; the bounded exact rerank serves that request meanwhile. Custom endpoints and explicit model overrides are left alone. To keep Jina, set `CODEDB_EMBEDDINGS_MODEL=jinaai/jina-embeddings-v2-base-code`. + +Local BM25/symbol retrieval, the watcher, and the installer are unchanged. + +## Verification + +```sh +zig build test --summary all +zig test src/test_semantic.zig +``` diff --git a/npm/package.json b/npm/package.json index 4ff4e1f9..ea54cf1f 100644 --- a/npm/package.json +++ b/npm/package.json @@ -1,6 +1,6 @@ { "name": "codedeebee", - "version": "0.2.5857", + "version": "0.2.5858", "description": "Zig code intelligence MCP server — npx launcher for the codedb native binary", "license": "MIT", "author": "justrach", diff --git a/src/release_info.zig b/src/release_info.zig index c356cb07..aaadeb11 100644 --- a/src/release_info.zig +++ b/src/release_info.zig @@ -1 +1 @@ -pub const semver = "0.2.5857"; +pub const semver = "0.2.5858"; diff --git a/src/semantic.zig b/src/semantic.zig index ac1c63b6..9556953e 100644 --- a/src/semantic.zig +++ b/src/semantic.zig @@ -14,7 +14,8 @@ const semantic_auth = @import("semantic_auth.zig"); pub const default_url = "https://embeddings.wiki.codes/v1/codedb/embeddings"; pub const legacy_qwen_model = "Qwen/Qwen3-Embedding-0.6B"; -pub const default_model = "jinaai/jina-embeddings-v2-base-code"; +pub const jina_model = "jinaai/jina-embeddings-v2-base-code"; +pub const default_model = "gemini-embedding-001"; pub const default_dimensions: u16 = 512; pub const default_timeout_ms: u32 = 15_000; pub const min_timeout_ms: u32 = 10; @@ -42,6 +43,7 @@ pub const default_ann_rrf_k: f32 = 1; pub const default_ann_semantic_weight: f32 = 2.5; pub const qwen_query_encoding_version = "qwen3-query-instruct-v1"; pub const jina_query_encoding_version = "jina-v2-code-symmetric-raw-v1"; +pub const gemini_query_encoding_version = "gemini001-code-retrieval-query-v1"; pub const document_card_version = "codedb-code-chunk-v2"; pub const calibration_text = "codedb vector-space calibration v1: deterministic code retrieval"; pub const calibration_min_cosine: f32 = 0.9999; @@ -146,18 +148,16 @@ pub const Config = struct { }; fn queryEncodingVersion(model: []const u8) []const u8 { - return if (std.mem.eql(u8, model, default_model)) - jina_query_encoding_version - else - qwen_query_encoding_version; + if (std.mem.eql(u8, model, jina_model)) return jina_query_encoding_version; + if (std.mem.eql(u8, model, default_model)) return gemini_query_encoding_version; + return qwen_query_encoding_version; } -/// Jina v2 code is a symmetric embedding model, so queries and code cards -/// share one raw-text vector space. Preserve the historical Qwen instruction -/// for explicit Qwen/custom-model overrides so upgrading CodeDB does not -/// silently change those deployments. +/// Jina v2 code is symmetric and keeps raw queries. The hosted Gemini edge +/// recognizes the existing instruction prefix as a CODE_RETRIEVAL_QUERY task; +/// preserve that prefix for explicit Qwen/custom-model overrides as well. fn formatQueryInput(allocator: std.mem.Allocator, model: []const u8, task: []const u8) ![]u8 { - if (std.mem.eql(u8, model, default_model)) return allocator.dupe(u8, task); + if (std.mem.eql(u8, model, jina_model)) return allocator.dupe(u8, task); return std.fmt.allocPrint( allocator, "Instruct: Retrieve code relevant to the user request.\nQuery: {s}", @@ -165,20 +165,22 @@ fn formatQueryInput(allocator: std.mem.Allocator, model: []const u8, task: []con ); } -test "semantic query encoding is raw for Jina and backward compatible for Qwen" { +test "semantic query encoding distinguishes Gemini, Jina and Qwen" { const testing = std.testing; const task = "find the authentication middleware"; - const jina = try formatQueryInput(testing.allocator, default_model, task); + const jina = try formatQueryInput(testing.allocator, jina_model, task); defer testing.allocator.free(jina); try testing.expectEqualStrings(task, jina); + const gemini = try formatQueryInput(testing.allocator, default_model, task); + defer testing.allocator.free(gemini); const qwen = try formatQueryInput(testing.allocator, legacy_qwen_model, task); defer testing.allocator.free(qwen); - try testing.expectEqualStrings( - "Instruct: Retrieve code relevant to the user request.\nQuery: find the authentication middleware", - qwen, - ); - try testing.expectEqualStrings(jina_query_encoding_version, queryEncodingVersion(default_model)); + const expected = "Instruct: Retrieve code relevant to the user request.\nQuery: find the authentication middleware"; + try testing.expectEqualStrings(expected, gemini); + try testing.expectEqualStrings(expected, qwen); + try testing.expectEqualStrings(gemini_query_encoding_version, queryEncodingVersion(default_model)); + try testing.expectEqualStrings(jina_query_encoding_version, queryEncodingVersion(jina_model)); try testing.expectEqualStrings(qwen_query_encoding_version, queryEncodingVersion(legacy_qwen_model)); } diff --git a/src/semantic_index.zig b/src/semantic_index.zig index b8658a66..43a805e8 100644 --- a/src/semantic_index.zig +++ b/src/semantic_index.zig @@ -1,8 +1,8 @@ //! Persistent, local code-chunk ANN index for hybrid codedb retrieval. //! //! Building is explicit with `codedb semantic-index`, except that a -//! sidecar produced by the former hosted Qwen default is migrated once in the -//! background when a hybrid query first encounters it. Safe, bounded file +//! sidecar produced by a former hosted Qwen or Jina default is migrated once +//! in the background when a hybrid query first encounters it. Safe, bounded file //! chunks are embedded remotely in batches, while the vectors, HNSW graph, and //! record mapping are written only to codedb's per-project local data dir. //! Querying sends only the user's bounded query and searches OpenPuffer in @@ -1005,21 +1005,20 @@ fn isLegacyHostedVectorSpace( stored_dimensions: u16, stored_vector_space_id: u64, ) bool { - // Automatic repository-wide embedding is deliberately limited to the one - // vector space shipped as CodeDB's hosted default before Jina. Custom URLs, - // custom model overrides, malformed metadata, and merely similar Qwen - // sidecars remain explicit user decisions. + // Only exact former hosted-default vector spaces may auto-migrate. + // Custom endpoints, model overrides and altered metadata remain explicit. if (!std.mem.eql(u8, current.url, semantic.default_url) or !std.mem.eql(u8, current.model, semantic.default_model) or current.dimensions != semantic.default_dimensions or - !std.mem.eql(u8, stored_model, semantic.legacy_qwen_model) or - stored_dimensions != semantic.default_dimensions) + stored_dimensions != semantic.default_dimensions or + (!std.mem.eql(u8, stored_model, semantic.legacy_qwen_model) and + !std.mem.eql(u8, stored_model, semantic.jina_model))) { return false; } const legacy = semantic.Config{ .url = semantic.default_url, - .model = semantic.legacy_qwen_model, + .model = stored_model, .token = null, .dimensions = semantic.default_dimensions, .timeout_ms = semantic.default_timeout_ms, @@ -1027,7 +1026,7 @@ fn isLegacyHostedVectorSpace( return stored_vector_space_id == legacy.vectorSpaceId(); } -/// Schedule a single transactional Qwen-to-Jina sidecar migration owned by +/// Schedule a single transactional former-default-to-Gemini sidecar migration owned by /// `cache`. The old generation stays referenced until build() atomically /// publishes the validated replacement. The exact semantic fallback can serve /// the triggering request while this worker runs. @@ -1284,7 +1283,7 @@ pub fn search( return searchLoaded(io, allocator, &loaded, task, k, load_ns, false); } -test "semantic ANN sidecar accepts Jina 512D and rejects a legacy Qwen vector space" { +test "semantic ANN sidecar accepts Gemini 512D and migrates exact prior hosted spaces" { const testing = std.testing; const io = testing.io; var tmp = testing.tmpDir(.{}); @@ -1340,6 +1339,14 @@ test "semantic ANN sidecar accepts Jina 512D and rejects a legacy Qwen vector sp semantic.default_dimensions, legacy_config.vectorSpaceId(), )); + var jina_config = config; + jina_config.model = semantic.jina_model; + try testing.expect(isLegacyHostedVectorSpace( + &config, + semantic.jina_model, + semantic.default_dimensions, + jina_config.vectorSpaceId(), + )); try testing.expect(!isLegacyHostedVectorSpace( &config, semantic.default_model, @@ -1375,10 +1382,10 @@ test "semantic ANN sidecar accepts Jina 512D and rejects a legacy Qwen vector sp io, testing.allocator, dir_path, - semantic.legacy_qwen_model, + semantic.jina_model, semantic.default_dimensions, 1234, - legacy_config.vectorSpaceId(), + jina_config.vectorSpaceId(), &calibration, null, slab_name,