fix(db): index the keyword projection by document, and drop the unread binary ANN indexes (#8105)

Author: waleedlatif1

packages/db/migrations/0372_drop_binary_search_indexes.sql

@@ -0,0 +1,18 @@
+-- Drops the five binary-quantized ANN indexes on "embedding_search". Ranking walks the
+-- half-precision graphs; no reader has ordered by a hamming distance on this projection since
+-- half-precision search shipped, so each index was maintained on every chunk write for nothing,
+-- and the populated one doubled the cost of filling the projection's source and ACL columns.
+-- The binary columns stay for now: the projection sync trigger still writes them.
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_search_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_search_384_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_search_768_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_search_1024_binary_hnsw_idx";--> statement-breakpoint
+-- migration-safe: DROP INDEX CONCURRENTLY IF EXISTS is idempotent on replay and takes no blocking lock.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_search_3072_binary_hnsw_idx";--> statement-breakpoint
+SET lock_timeout = '5s';

packages/db/migrations/0373_keyword_tin_document_index.sql

@@ -0,0 +1,8 @@
+-- The document ACL trigger fans a document's source and ACL out to the keyword projection by
+-- document id; with no index on that column every ACL change scanned the whole projection.
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+-- migration-safe: replay replaces only this new index to recover an interrupted concurrent build; existing indexes remain available.
+DROP INDEX CONCURRENTLY IF EXISTS "embedding_keyword_tin_document_idx";--> statement-breakpoint
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "embedding_keyword_tin_document_idx" ON "embedding_keyword_tin" USING btree ("document_id");--> statement-breakpoint
+SET lock_timeout = '5s';