improvement(tables): select ids only for capped deletes and gate only trigger-dependent tests

Author: waleedlatif1

apps/sim/lib/table/rows/row-writes.integration.ts

@@ -1,7 +1,8 @@
 /**
  * Row-write integration tests against the provisioned, disposable TEST_DATABASE_URL database (the
- * integration setup points DATABASE_URL at it too). The migrated `rows_version` triggers, the
- * row-order advisory lock, and the filtered bulk-mutation queries all run for real.
+ * integration setup points DATABASE_URL at it too). The row-order advisory lock and the filtered
+ * bulk-mutation queries run for real everywhere; the `rows_version` cases also need the migrated
+ * triggers and skip on a `db:push` schema.
  */
 import { db } from '@sim/db'
 import { userTableDefinitions, userTableRows } from '@sim/db/schema'
@@ -44,11 +45,15 @@ const control = postgres(url, { max: 4, onnotice: () => {} })
 const workspaceId = generateId()
 const userId = generateId()
 
-/** `db:push` never installs the migration-only triggers these assertions depend on. */
-const [{ migrated }] = await control<{ migrated: boolean }[]>`SELECT EXISTS (
-  SELECT 1 FROM pg_trigger WHERE tgname = 'user_table_rows_version_update_trigger'
-) AS migrated`
-if (!migrated) await control.end()
+/**
+ * `db:push` never installs migration-only objects: the `rows_version` triggers and the
+ * `(workspace_id, table_id)` dependency statistic. Only the cases that depend on them need them.
+ */
+const [{ migrated }] = await control<{ migrated: boolean }[]>`SELECT
+  EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 'user_table_rows_version_update_trigger')
+  AND EXISTS (
+    SELECT 1 FROM pg_statistic_ext WHERE stxname = 'user_table_rows_workspace_table_stats'
+  ) AS migrated`
 
 async function createTable(columns: ColumnDefinition[]): Promise<TableDefinition> {
   const id = generateId()
@@ -85,7 +90,7 @@ function planNodes(plan: QueryPlan): QueryPlan[] {
 const textColumns = (...ids: string[]): ColumnDefinition[] =>
   ids.map((id) => ({ id, name: id, type: 'string' }))
 
-describe.skipIf(!migrated)('table row writes against real PostgreSQL', () => {
+describe('table row writes against real PostgreSQL', () => {
   beforeAll(async () => {
     await control`INSERT INTO "user" (id, name, email, email_verified, created_at, updated_at)
       VALUES (${userId}, 'Row write fixture', ${`${userId}@example.test`}, true, now(), now())`
@@ -103,7 +108,7 @@ describe.skipIf(!migrated)('table row writes against real PostgreSQL', () => {
     await control.end()
   })
 
-  describe('rows_version', () => {
+  describe.skipIf(!migrated)('rows_version', () => {
     it('advances once for a transaction that edits cells across several statements', async () => {
       const table = await createTable(textColumns('name'))
       await seedRows(table.id, [
@@ -379,7 +384,8 @@ describe.skipIf(!migrated)('table row writes against real PostgreSQL', () => {
       expect([...result.affectedRowIds].sort()).toEqual([...expected].sort())
     })
 
-    it('fences only a containment filter into a GIN probe and leaves any other filter unfenced', async () => {
+    /** A table large enough that the planner's index choice reflects its statistics. */
+    async function seedPlanFixture(): Promise<(filter: Filter) => Promise<QueryPlan[]>> {
       const table = await createTable(textColumns('kind', 'n'))
       await control`INSERT INTO user_table_rows (id, table_id, workspace_id, data, order_key)
         SELECT ${table.id} || '-' || n, ${table.id}, ${workspaceId},
@@ -388,7 +394,7 @@ describe.skipIf(!migrated)('table row writes against real PostgreSQL', () => {
         FROM generate_series(1, 20000) AS n`
       await control`ANALYZE user_table_rows`
 
-      async function planFor(filter: Filter): Promise<QueryPlan[]> {
+      return async (filter) => {
         const filterClause = buildFilterClause(
           filter,
           USER_TABLE_ROWS_SQL_NAME,
@@ -397,25 +403,35 @@ describe.skipIf(!migrated)('table row writes against real PostgreSQL', () => {
         if (!filterClause) throw new Error('Fixture filter compiled to no clause')
         const plans = await withSeqscanOff((trx) =>
           trx.execute<{ 'QUERY PLAN': { Plan: QueryPlan }[] }>(
-            sql`EXPLAIN (FORMAT JSON) ${firstMatchingRowsQuery(table, filter, filterClause, 5)}`
+            sql`EXPLAIN (FORMAT JSON) ${firstMatchingRowsQuery(table, filter, filterClause, 5, 'rows')}`
           )
         )
         return planNodes(plans[0]['QUERY PLAN'][0].Plan)
       }
+    }
+
+    it('fences only a containment filter and leaves any other filter unfenced', async () => {
+      const planFor = await seedPlanFixture()
 
       const fenced = await planFor({ kind: 'hit' })
       expect(fenced.some((node) => node['Node Type'] === 'CTE Scan')).toBe(true)
+
+      const walked = await planFor({ kind: { $ne: 'miss' } })
+      expect(walked.some((node) => node['Node Type'] === 'CTE Scan')).toBe(false)
+      expect(walked[0]['Node Type']).toBe('Limit')
+    })
+
+    it.skipIf(!migrated)('probes the tenant GIN index inside the fence', async () => {
+      const planFor = await seedPlanFixture()
+
+      const fenced = await planFor({ kind: 'hit' })
       expect(
         fenced.some(
           (node) =>
             node['Node Type'] === 'Bitmap Index Scan' &&
             node['Index Name'] === 'user_table_rows_tenant_data_gin_idx'
         )
       ).toBe(true)
-
-      const walked = await planFor({ kind: { $ne: 'miss' } })
-      expect(walked.some((node) => node['Node Type'] === 'CTE Scan')).toBe(false)
-      expect(walked[0]['Node Type']).toBe('Limit')
     })
   })
 

apps/sim/lib/table/rows/service.ts

@@ -982,7 +982,8 @@ function buildRowOrderBySql(
 }
 
 /**
- * The first `limit` rows matching `filter` in the default `(order_key, id)` order.
+ * The first `limit` rows matching `filter` in the default `(order_key, id)` order, projected to
+ * their ids alone or to `id` and `data`.
  *
  * As one ordered query, the planner walks the `(table_id, order_key, id)` index and tests each row
  * against the filter until `limit` match: cheap when matches are dense, the whole table when they
@@ -995,14 +996,19 @@ export function firstMatchingRowsQuery(
   table: TableDefinition,
   filter: Filter,
   filterClause: SQL,
-  limit: number
+  limit: number,
+  projection: 'ids' | 'rows'
 ): SQL {
+  const columns =
+    projection === 'rows'
+      ? sql`${userTableRows.id}, ${userTableRows.data}`
+      : sql`${userTableRows.id}`
   const tenantMatch = sql`${userTableRows.tableId} = ${table.id}
     AND ${userTableRows.workspaceId} = ${table.workspaceId}
     AND ${filterClause}`
   if (!isContainmentOnlyFilter(filter, table.schema.columns)) {
     return sql`
-      SELECT ${userTableRows.id}, ${userTableRows.data}
+      SELECT ${columns}
       FROM ${userTableRows}
       WHERE ${tenantMatch}
       ORDER BY ${userTableRows.orderKey}, ${userTableRows.id}
@@ -1018,7 +1024,7 @@ export function firstMatchingRowsQuery(
     kept AS (
       SELECT id, order_key FROM matched ORDER BY order_key, id LIMIT ${limit}
     )
-    SELECT ${userTableRows.id}, ${userTableRows.data}
+    SELECT ${columns}
     FROM kept
     INNER JOIN ${userTableRows} ON ${userTableRows.id} = kept.id
     ORDER BY kept.order_key, kept.id
@@ -1029,11 +1035,26 @@ async function selectFirstMatchingRows(
   table: TableDefinition,
   filter: Filter,
   filterClause: SQL,
-  limit: number
-): Promise<Array<{ id: string; data: RowData }>> {
+  limit: number,
+  projection: 'ids'
+): Promise<Array<{ id: string }>>
+async function selectFirstMatchingRows(
+  table: TableDefinition,
+  filter: Filter,
+  filterClause: SQL,
+  limit: number,
+  projection: 'rows'
+): Promise<Array<{ id: string; data: RowData }>>
+async function selectFirstMatchingRows(
+  table: TableDefinition,
+  filter: Filter,
+  filterClause: SQL,
+  limit: number,
+  projection: 'ids' | 'rows'
+): Promise<Array<{ id: string; data?: RowData }>> {
   const rows = await withSeqscanOff(async (trx) =>
-    trx.execute<{ id: string; data: RowData }>(
-      firstMatchingRowsQuery(table, filter, filterClause, limit)
+    trx.execute<{ id: string; data?: RowData }>(
+      firstMatchingRowsQuery(table, filter, filterClause, limit, projection)
     )
   )
   return Array.from(rows)
@@ -2370,7 +2391,13 @@ export async function updateRowsByFilter(
     return { affectedCount: affectedRowIds.length, affectedRowIds }
   }
 
-  const matchingRows = await selectFirstMatchingRows(table, data.filter, filterClause, limit)
+  const matchingRows = await selectFirstMatchingRows(
+    table,
+    data.filter,
+    filterClause,
+    limit,
+    'rows'
+  )
   if (matchingRows.length === 0) {
     return { affectedCount: 0, affectedRowIds: [] }
   }
@@ -2745,7 +2772,13 @@ export async function deleteRowsByFilter(
       if (page.length < TABLE_LIMITS.DELETE_PAGE_SIZE) break
     }
   } else {
-    const matchingRows = await selectFirstMatchingRows(table, data.filter, filterClause, limit)
+    const matchingRows = await selectFirstMatchingRows(
+      table,
+      data.filter,
+      filterClause,
+      limit,
+      'ids'
+    )
     const rowIds = matchingRows.map((row) => row.id)
     if (rowIds.length > 0) {
       const deletedIds = await deleteOrderedRowsByIds({