From 61a7bc10da85ff75151364718a5695e8d1e6014c Mon Sep 17 00:00:00 2001 From: xiaocui-big <1057059996@qq.com> Date: Thu, 17 Sep 2026 13:11:43 +0800 Subject: [PATCH 1/6] perf(migrations): add HNSW ANN indexes for embedding tables Add pgvector HNSW indexes on embeddings_text (768-d), embeddings_visual (512-d), and embeddings_face (512-d) using vector_cosine_ops to match the <=> cosine distance operator used by search and relator queries. Previously all vector queries performed exact sequential scans of the full embedding table. The dimension blocker cited in the original DDL comments has been resolved: all three tables now use fixed-dimension columns declared in the schema. Resolves: #173 --- .../migrations/0024_add_hnsw_ann_indexes.sql | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql diff --git a/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql b/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql new file mode 100644 index 0000000..99eb2bf --- /dev/null +++ b/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql @@ -0,0 +1,34 @@ +-- +goose Up +-- Add HNSW ANN indexes for all embedding tables so vector queries use +-- approximate nearest-neighbor search instead of exact sequential scans. +-- All three tables have fixed-dimension columns (768 for text, 512 for +-- visual and face), and all queries use cosine distance (<=>). +-- +-- pgvector HNSW defaults: m=16, ef_construction=64. These are suitable +-- for the personal-corpus scale this server targets. Operators class +-- vector_cosine_ops matches the <=> distance operator used by search +-- and relator queries. +-- +-- Resolves: https://github.com/bytefolk/mem/issues/173 + +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_text_embedding_hnsw + ON embeddings_text USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_visual_embedding_hnsw + ON embeddings_visual USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_face_embedding_hnsw + ON embeddings_face USING hnsw (embedding vector_cosine_ops); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_embeddings_face_embedding_hnsw; +DROP INDEX IF EXISTS idx_embeddings_visual_embedding_hnsw; +DROP INDEX IF EXISTS idx_embeddings_text_embedding_hnsw; +-- +goose StatementEnd From c96424f65577139b2053b852a851f531cbc221d5 Mon Sep 17 00:00:00 2001 From: xiaocui-big <1057059996@qq.com> Date: Thu, 17 Sep 2026 13:12:19 +0800 Subject: [PATCH 2/6] chore(migrations): remove stale HNSW deferral comment from 0001_init The dimension blocker cited in this comment has been resolved: embeddings_text uses a fixed vector(768) column. The HNSW index is now created by migration 0024. Refs: #173 --- server/internal/db/migrations/0001_init.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/internal/db/migrations/0001_init.sql b/server/internal/db/migrations/0001_init.sql index a5f9bf2..8776c00 100644 --- a/server/internal/db/migrations/0001_init.sql +++ b/server/internal/db/migrations/0001_init.sql @@ -106,8 +106,6 @@ CREATE TABLE IF NOT EXISTS embeddings_text ( embedding vector(768) ); CREATE INDEX IF NOT EXISTS idx_embeddings_text_file ON embeddings_text (file_id); --- HNSW index will be added by worker once we settle on a model dimension. Kept off here --- because pgvector requires the table to have data of consistent dim before building. -- +goose StatementEnd -- +goose StatementBegin From 0824ef34c894e50373874f03974402f3966698ac Mon Sep 17 00:00:00 2001 From: xiaocui-big <1057059996@qq.com> Date: Thu, 17 Sep 2026 13:12:54 +0800 Subject: [PATCH 3/6] chore(migrations): clarify ANN deferral scope in 0019 Scope the dimension-specific ANN guidance to index_generation_vectors explicitly, and note that the legacy embedding tables now have HNSW indexes (migration 0024). The versioned table index is deferred until the generation executor exists. Refs: #173 --- .../db/migrations/0019_versioned_index_generations.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/internal/db/migrations/0019_versioned_index_generations.sql b/server/internal/db/migrations/0019_versioned_index_generations.sql index bd173c8..72003bd 100644 --- a/server/internal/db/migrations/0019_versioned_index_generations.sql +++ b/server/internal/db/migrations/0019_versioned_index_generations.sql @@ -264,8 +264,10 @@ CREATE INDEX idx_index_generation_targets_file_hash -- +goose StatementBegin -- `vector` intentionally has no table-wide dimension. Every row is validated -- against its immutable generation.output_dimension by the canonical service. --- Future ANN indexes must be route/dimension-specific expression or partition --- indexes; silently padding or truncating vectors is never allowed. +-- ANN indexes on this table must be route/dimension-specific expression or +-- partition indexes; silently padding or truncating vectors is never allowed. +-- The legacy embeddings_text/visual/face tables have HNSW indexes (migration +-- 0024); this versioned table is deferred until the generation executor exists. CREATE TABLE index_generation_vectors ( generation_id uuid NOT NULL REFERENCES index_generations(id) ON DELETE CASCADE, workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, From 8e938cad4d81683d18b4f833a6980392d2641f2f Mon Sep 17 00:00:00 2001 From: xiaocui-big <1057059996@qq.com> Date: Thu, 17 Sep 2026 13:14:17 +0800 Subject: [PATCH 4/6] fix(migrations): run HNSW index creation outside transaction CREATE INDEX CONCURRENTLY cannot execute inside a transaction block. Add goose NO TRANSACTION annotation so the migration runs without transaction wrapping, allowing concurrent index builds that do not block writes on populated embedding tables. Refs: #173 --- server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql b/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql index 99eb2bf..8c2fe45 100644 --- a/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql +++ b/server/internal/db/migrations/0024_add_hnsw_ann_indexes.sql @@ -1,3 +1,4 @@ +-- +goose NO TRANSACTION -- +goose Up -- Add HNSW ANN indexes for all embedding tables so vector queries use -- approximate nearest-neighbor search instead of exact sequential scans. @@ -9,6 +10,9 @@ -- vector_cosine_ops matches the <=> distance operator used by search -- and relator queries. -- +-- CONCURRENTLY requires running outside a transaction block, hence the +-- NO TRANSACTION annotation above. +-- -- Resolves: https://github.com/bytefolk/mem/issues/173 -- +goose StatementBegin From 3c38b7250cf94624f6be8b04af6b16aa4d82159b Mon Sep 17 00:00:00 2001 From: xiaocui-big <1057059996@qq.com> Date: Thu, 17 Sep 2026 13:23:18 +0800 Subject: [PATCH 5/6] chore(verify): bump EXPECTED_MIGRATION_HEAD to 24 Migration 0024 adds HNSW ANN indexes for the three embedding tables. Refs: #173 --- scripts/verify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/verify.sh b/scripts/verify.sh index eed61b0..79c51d9 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -4,7 +4,7 @@ set -euo pipefail REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" MODE="${1:-unit}" -EXPECTED_MIGRATION_HEAD=23 +EXPECTED_MIGRATION_HEAD=24 MIGRATION_ROLLBACK_TARGET=11 MODEL_TEXT_CANONICAL_BASE=15 WORKSPACE_AI_PROFILE_BASE=16 From 329307fc06f7ab19233111d0ffd496420d0d43c4 Mon Sep 17 00:00:00 2001 From: xiaocui-big <1057059996@qq.com> Date: Thu, 17 Sep 2026 13:23:28 +0800 Subject: [PATCH 6/6] chore(changelog): add HNSW ANN index entry under [Unreleased] Refs: #173 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a5e985..435f8dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ The project publishes 0.x prerelease versions; a stable release line is not yet ### Added +- HNSW ANN indexes for `embeddings_text`, `embeddings_visual`, and + `embeddings_face` (migration 0024, `#173`). Vector search queries now use + approximate nearest-neighbor via pgvector `vector_cosine_ops` instead of + exact sequential scans. - `mem doctor` — a read-only diagnosis of why the CLI cannot talk to a working server (`#112`). It reports four checks in a fixed order: reachability of the configured server URL, whether a credential exists, the workspace the server