diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a5e985..435f8dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ The project publishes 0.x prerelease versions; a stable release line is not yet ### Added +- HNSW ANN indexes for `embeddings_text`, `embeddings_visual`, and + `embeddings_face` (migration 0024, `#173`). Vector search queries now use + approximate nearest-neighbor via pgvector `vector_cosine_ops` instead of + exact sequential scans. - `mem doctor` — a read-only diagnosis of why the CLI cannot talk to a working server (`#112`). It reports four checks in a fixed order: reachability of the configured server URL, whether a credential exists, the workspace the server diff --git a/scripts/verify.sh b/scripts/verify.sh index eed61b0..79c51d9 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -4,7 +4,7 @@ set -euo pipefail REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" MODE="${1:-unit}" -EXPECTED_MIGRATION_HEAD=23 +EXPECTED_MIGRATION_HEAD=24 MIGRATION_ROLLBACK_TARGET=11 MODEL_TEXT_CANONICAL_BASE=15 WORKSPACE_AI_PROFILE_BASE=16 diff --git a/server/internal/db/migrations/0001_init.sql b/server/internal/db/migrations/0001_init.sql index a5f9bf2..8776c00 100644 --- a/server/internal/db/migrations/0001_init.sql +++ b/server/internal/db/migrations/0001_init.sql @@ -106,8 +106,6 @@ CREATE TABLE IF NOT EXISTS embeddings_text ( embedding vector(768) ); CREATE INDEX IF NOT EXISTS idx_embeddings_text_file ON embeddings_text (file_id); --- HNSW index will be added by worker once we settle on a model dimension. Kept off here --- because pgvector requires the table to have data of consistent dim before building. -- +goose StatementEnd -- +goose StatementBegin diff --git a/server/internal/db/migrations/0019_versioned_index_generations.sql b/server/internal/db/migrations/0019_versioned_index_generations.sql index bd173c8..72003bd 100644 --- a/server/internal/db/migrations/0019_versioned_index_generations.sql +++ b/server/internal/db/migrations/0019_versioned_index_generations.sql @@ -264,8 +264,10 @@ CREATE INDEX idx_index_generation_targets_file_hash -- +goose StatementBegin -- `vector` intentionally has no table-wide dimension. Every row is validated -- against its immutable generation.output_dimension by the canonical service. --- Future ANN indexes must be route/dimension-specific expression or partition --- indexes; silently padding or truncating vectors is never allowed. +-- ANN indexes on this table must be route/dimension-specific expression or +-- partition indexes; silently padding or truncating vectors is never allowed. +-- The legacy embeddings_text/visual/face tables have HNSW indexes (migration +-- 0024); this versioned table is deferred until the generation executor exists. CREATE TABLE index_generation_vectors ( generation_id uuid NOT NULL REFERENCES index_generations(id) ON DELETE CASCADE, workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, diff --git a/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql b/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql new file mode 100644 index 0000000..8c2fe45 --- /dev/null +++ b/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql @@ -0,0 +1,38 @@ +-- +goose NO TRANSACTION +-- +goose Up +-- Add HNSW ANN indexes for all embedding tables so vector queries use +-- approximate nearest-neighbor search instead of exact sequential scans. +-- All three tables have fixed-dimension columns (768 for text, 512 for +-- visual and face), and all queries use cosine distance (<=>). +-- +-- pgvector HNSW defaults: m=16, ef_construction=64. These are suitable +-- for the personal-corpus scale this server targets. Operators class +-- vector_cosine_ops matches the <=> distance operator used by search +-- and relator queries. +-- +-- CONCURRENTLY requires running outside a transaction block, hence the +-- NO TRANSACTION annotation above. +-- +-- Resolves: https://github.com/bytefolk/mem/issues/173 + +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_text_embedding_hnsw + ON embeddings_text USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_visual_embedding_hnsw + ON embeddings_visual USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_face_embedding_hnsw + ON embeddings_face USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_embeddings_face_embedding_hnsw; +DROP INDEX IF EXISTS idx_embeddings_visual_embedding_hnsw; +DROP INDEX IF EXISTS idx_embeddings_text_embedding_hnsw; +-- +goose StatementEnd