chore(db): drop unused ANN indexes on embedding (#7955)

* chore(db): drop unused ANN indexes on embedding

Approximate retrieval is served by the compact embedding_search projection.
The only vector ordering left on embedding is an exact rerank wrapped as
(distance) + 0, which the planner cannot match to an index expression, and
the binary_quantize expression indexes were never referenced by any query.
The last app version that ordered by a bare distance has drained, so these
ten indexes were maintained on every chunk write while never being scanned.

* fix(knowledge): let the scale benchmark own its ANN index

The bulk-seed path required embedding_vector_hnsw_idx to exist, dropped it
before seeding and rebuilt it after. With no ANN index left in the schema it
threw before measuring anything. The benchmark seeds embedding directly and
never populates embedding_search, so it now creates its own HNSW index after
the load rather than borrowing a schema one, for both seed paths.

Author: waleedlatif1

apps/sim/lib/knowledge/__integration__/scale.integration.ts

@@ -40,6 +40,15 @@ const rows = Number(process.env.KNOWLEDGE_SCALE_DOCUMENTS ?? 250_000)
 const SEED_BATCH_SIZE = 2_000
 const PAGE_SIZE = 500
 const DIMENSIONS = 1536
+/**
+ * `embedding` carries no ANN index in the schema — production serves approximate
+ * retrieval from the `embedding_search` projection, which this fixture does not
+ * populate. The benchmark still measures ANN behaviour over the dense corpus it
+ * seeds directly, so it owns this index rather than borrowing a schema one, and
+ * builds it after the load instead of paying index maintenance on every insert.
+ */
+const BENCHMARK_VECTOR_INDEX = 'embedding_scale_benchmark_hnsw_idx'
+const BENCHMARK_VECTOR_INDEX_DEFINITION = `CREATE INDEX IF NOT EXISTS ${BENCHMARK_VECTOR_INDEX} ON public.embedding USING hnsw (embedding vector_cosine_ops) WITH (m='16', ef_construction='64')`
 const logger = createLogger('KnowledgeScaleIntegration')
 if (reuseReportFile && statSync(reuseReportFile).size > 16 * 1024 * 1024)
   throw new Error('Retained scale report must be at most 16 MiB')
@@ -386,7 +395,6 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide
   it.skipIf(metadataOnly)(
     'stores a bounded dense corpus and measures ACL/tag-filtered vector and hybrid retrieval',
     async () => {
-      let vectorIndexDefinition: string | undefined
       if (bulkSeed) {
         const other = await db
           .select({ id: embedding.id })
@@ -397,13 +405,6 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide
           throw new Error(
             'Bulk scale setup requires a database containing only its own fixture chunks'
           )
-        const [index] = await db.execute(
-          sql`SELECT indexdef FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'embedding_vector_hnsw_idx'`
-        )
-        if (typeof index?.indexdef !== 'string')
-          throw new Error('Canonical 1536-dimensional HNSW index is missing')
-        vectorIndexDefinition = index.indexdef
-        await db.execute(sql`DROP INDEX embedding_vector_hnsw_idx`)
       }
       if (!reuseReportFile)
         await measure('seed.vectors', async () => {
@@ -420,20 +421,17 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide
             }
           }
         })
-      if (vectorIndexDefinition) {
-        const definition = vectorIndexDefinition
-        await measure('seed.hnswBuild', () =>
-          db.transaction(async (tx) => {
-            await tx.execute(sql`SET LOCAL maintenance_work_mem = '2GB'`)
-            await tx.execute(sql`SET LOCAL max_parallel_maintenance_workers = 2`)
-            await tx.execute(sql.raw(definition))
-          })
-        )
-        const [restored] = await db.execute(
-          sql`SELECT indexdef FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'embedding_vector_hnsw_idx'`
-        )
-        expect(restored.indexdef).toBe(vectorIndexDefinition)
-      }
+      await measure('seed.hnswBuild', () =>
+        db.transaction(async (tx) => {
+          await tx.execute(sql`SET LOCAL maintenance_work_mem = '2GB'`)
+          await tx.execute(sql`SET LOCAL max_parallel_maintenance_workers = 2`)
+          await tx.execute(sql.raw(BENCHMARK_VECTOR_INDEX_DEFINITION))
+        })
+      )
+      const [built] = await db.execute(
+        sql`SELECT indexdef FROM pg_indexes WHERE schemaname = 'public' AND indexname = ${BENCHMARK_VECTOR_INDEX}`
+      )
+      expect(built?.indexdef).toEqual(expect.stringContaining('USING hnsw'))
       await db.execute(sql`ANALYZE embedding`)
       await db.execute(sql`ANALYZE document`)
       const [count] = await db.execute(

apps/sim/lib/knowledge/vector-columns.ts

@@ -58,16 +58,17 @@ export function embeddingVectorValues(
 }
 
 /**
- * Cosine distance between a chunk's vector and the query vector, in the exact
- * form the width's HNSW index was built on.
+ * Cosine distance between a chunk's vector and the query vector, used for the
+ * search layer's exact rerank over an already-bounded candidate set.
  *
- * The 3,072 column is compared through a `halfvec` cast because pgvector
- * indexes `vector` only up to 2,000 dimensions, so its index is on that cast
- * expression. Postgres matches an expression index by the expression, so a
- * plain `<=>` against the column here would silently drop to a sequential scan
- * — and the cast belongs here rather than at each call site precisely because
- * getting it wrong is invisible in the results and only shows up as latency.
- * `packages/db/schema.ts` records what the half-precision comparison costs.
+ * `embedding` carries no ANN index — approximate retrieval runs against the
+ * compact `embedding_search` projection — so this expression is never expected
+ * to match one, and its caller wraps it to keep the planner from trying.
+ *
+ * The 3,072 column keeps its `halfvec` cast: the width that made the cast
+ * necessary is unchanged, and `packages/db/schema.ts` records that the
+ * half-precision comparison moved no distance measurably for the one model
+ * that emits this width.
  */
 export function embeddingDistance(
   dimensions: KbEmbeddingDimensions,

packages/db/migrations/0360_drop_unused_embedding_hnsw_indexes.sql

@@ -0,0 +1,27 @@
+-- Drops the ten unused ANN indexes on "embedding". Approximate retrieval moved to the
+-- compact "embedding_search" projection; the only vector ordering left on this table is an
+-- exact rerank wrapped as (distance) + 0, which the planner cannot match to an index.
+-- The last app version that ordered by a bare distance has drained, so nothing reads these.
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_384_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_768_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_1024_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_3072_vector_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_384_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_768_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_1024_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_3072_binary_hnsw_idx";--> statement-breakpoint
+SET lock_timeout = '5s';