perf(migrations): add HNSW ANN indexes for embedding tables

CHANGELOG.md

@@ -9,6 +9,10 @@ The project publishes 0.x prerelease versions; a stable release line is not yet
 
 ### Added
 
+- HNSW ANN indexes for `embeddings_text`, `embeddings_visual`, and
+  `embeddings_face` (migration 0024, `#173`). Vector search queries now use
+  approximate nearest-neighbor via pgvector `vector_cosine_ops` instead of
+  exact sequential scans.
 - `mem doctor` — a read-only diagnosis of why the CLI cannot talk to a working
   server (`#112`). It reports four checks in a fixed order: reachability of the
   configured server URL, whether a credential exists, the workspace the server

scripts/verify.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
 MODE="${1:-unit}"
-EXPECTED_MIGRATION_HEAD=23
+EXPECTED_MIGRATION_HEAD=24
 MIGRATION_ROLLBACK_TARGET=11
 MODEL_TEXT_CANONICAL_BASE=15
 WORKSPACE_AI_PROFILE_BASE=16

server/internal/db/migrations/0001_init.sql

@@ -106,8 +106,6 @@ CREATE TABLE IF NOT EXISTS embeddings_text (
     embedding       vector(768)
 );
 CREATE INDEX IF NOT EXISTS idx_embeddings_text_file ON embeddings_text (file_id);
--- HNSW index will be added by worker once we settle on a model dimension. Kept off here
--- because pgvector requires the table to have data of consistent dim before building.
 -- +goose StatementEnd
 
 -- +goose StatementBegin

server/internal/db/migrations/0019_versioned_index_generations.sql

@@ -264,8 +264,10 @@ CREATE INDEX idx_index_generation_targets_file_hash
 -- +goose StatementBegin
 -- `vector` intentionally has no table-wide dimension. Every row is validated
 -- against its immutable generation.output_dimension by the canonical service.
--- Future ANN indexes must be route/dimension-specific expression or partition
--- indexes; silently padding or truncating vectors is never allowed.
+-- ANN indexes on this table must be route/dimension-specific expression or
+-- partition indexes; silently padding or truncating vectors is never allowed.
+-- The legacy embeddings_text/visual/face tables have HNSW indexes (migration
+-- 0024); this versioned table is deferred until the generation executor exists.
 CREATE TABLE index_generation_vectors (
     generation_id           uuid NOT NULL REFERENCES index_generations(id) ON DELETE CASCADE,
     workspace_id            uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,

server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql

@@ -0,0 +1,38 @@
+-- +goose NO TRANSACTION
+-- +goose Up
+-- Add HNSW ANN indexes for all embedding tables so vector queries use
+-- approximate nearest-neighbor search instead of exact sequential scans.
+-- All three tables have fixed-dimension columns (768 for text, 512 for
+-- visual and face), and all queries use cosine distance (<=>).
+--
+-- pgvector HNSW defaults: m=16, ef_construction=64. These are suitable
+-- for the personal-corpus scale this server targets. Operators class
+-- vector_cosine_ops matches the <=> distance operator used by search
+-- and relator queries.
+--
+-- CONCURRENTLY requires running outside a transaction block, hence the
+-- NO TRANSACTION annotation above.
+--
+-- Resolves: https://github.com/bytefolk/mem/issues/173
+
+-- +goose StatementBegin
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_text_embedding_hnsw
+    ON embeddings_text USING hnsw (embedding vector_cosine_ops);
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_visual_embedding_hnsw
+    ON embeddings_visual USING hnsw (embedding vector_cosine_ops);
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_face_embedding_hnsw
+    ON embeddings_face USING hnsw (embedding vector_cosine_ops);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP INDEX IF EXISTS idx_embeddings_face_embedding_hnsw;
+DROP INDEX IF EXISTS idx_embeddings_visual_embedding_hnsw;
+DROP INDEX IF EXISTS idx_embeddings_text_embedding_hnsw;
+-- +goose StatementEnd