fix(knowledge): skip global readiness checks for ordinary KBs

Author: TheodoreSpeaks

apps/sim/lib/knowledge/search/queries.test.ts

@@ -438,6 +438,26 @@ describe('workspace-scoped vector retrieval', () => {
     vi.useRealTimers()
   })
 
+  it.each([false, undefined])(
+    'retrieves ordinary KB results without global projection readiness (searchIndexOnly=%s)',
+    async (searchIndexOnly) => {
+      queueTableRows(schemaMock.embedding, [...ranked].reverse())
+      const result = await retrieveKnowledgeSearch({
+        ...params,
+        searchIndexOnly,
+        searchMode: 'vector',
+        query: 'What is the capital of France?',
+      })
+      expect(result.retrieval).toEqual({ status: 'complete', timedOutLegs: [] })
+      expect(result.rows.map((row) => row.id)).toEqual(['near', 'far'])
+      expect(statements().filter((query) => query.sql.includes('AS unfilled'))).toHaveLength(0)
+      expect(statements().some((query) => isPageStatement(query.sql))).toBe(true)
+      const walk = statements().find((query) => isWalk(query.sql))!
+      expect(JSON.stringify(walk)).toContain('required_clause')
+      expect(JSON.stringify(walk)).toContain(String(schemaMock.document.acl))
+    }
+  )
+
   it.each([handleVectorOnlySearch, handleTagAndVectorSearch])(
     'does not acquire a connection or start SQL after the KB retrieval deadline',
     async (search) => {
@@ -749,7 +769,7 @@ describe('workspace-scoped vector retrieval', () => {
       statements()
         .filter((query) => query.sql.includes('statement_timeout'))
         .map((query) => query.params[0])
-    ).toEqual(['100', '100', '40', '20'])
+    ).toEqual(['100', '40', '20'])
   })
 
   it('applies the scan settings in the deadline statement rather than one of their own', async () => {
@@ -805,49 +825,53 @@ describe('workspace-scoped vector retrieval', () => {
     )
   })
 
-  it('reports incomplete retrieval for 18 expired pool waiters without starting their SQL later', async () => {
-    vi.useFakeTimers()
-    const release: Array<() => void> = []
-    const transactions: Array<Promise<unknown>> = []
-    vi.spyOn(db, 'transaction').mockImplementation((callback) => {
-      const transaction = new Promise<void>((resolve) => release.push(resolve)).then(() =>
-        callback(db as never)
+  it.each([false, true])(
+    'expires pool waiters without starting SQL later (searchIndexOnly=%s)',
+    async (searchIndexOnly) => {
+      vi.useFakeTimers()
+      const release: Array<() => void> = []
+      const transactions: Array<Promise<unknown>> = []
+      vi.spyOn(db, 'transaction').mockImplementation((callback) => {
+        const transaction = new Promise<void>((resolve) => release.push(resolve)).then(() =>
+          callback(db as never)
+        )
+        transactions.push(transaction)
+        return transaction as ReturnType<typeof db.transaction>
+      })
+      const pending = Promise.all(
+        Array.from({ length: 18 }, (_, index) =>
+          retrieveKnowledgeSearch({
+            ...params,
+            knowledgeBaseIds: [`kb-${index}`],
+            searchIndexOnly,
+            query: 'fixture policy',
+            searchMode: 'vector',
+            vectorBudgetMs: 50,
+          })
+        )
       )
-      transactions.push(transaction)
-      return transaction as ReturnType<typeof db.transaction>
-    })
-    const pending = Promise.all(
-      Array.from({ length: 18 }, (_, index) =>
-        retrieveKnowledgeSearch({
-          ...params,
-          knowledgeBaseIds: [`kb-${index}`],
-          query: 'fixture policy',
-          searchMode: 'vector',
-          vectorBudgetMs: 50,
+      await vi.advanceTimersByTimeAsync(60)
+      const results = await pending
+      expect(results).toHaveLength(18)
+      for (const result of results) {
+        expect(result).toEqual({
+          rows: [],
+          retrieval: { status: 'partial', timedOutLegs: ['vector'] },
         })
-      )
-    )
-    await vi.advanceTimersByTimeAsync(60)
-    const results = await pending
-    expect(results).toHaveLength(18)
-    for (const result of results) {
-      expect(result).toEqual({
-        rows: [],
-        retrieval: { status: 'partial', timedOutLegs: ['vector'] },
-      })
-    }
-    for (const resume of release) resume()
-    const settled = await Promise.allSettled(transactions)
-    /** The searches that miss the projection-fill memo together share one read; each search's own read is refused at its deadline before it starts. */
-    expect(settled).toHaveLength(1)
-    for (const transaction of settled) {
-      expect(transaction.status).toBe('rejected')
-      if (transaction.status === 'rejected')
-        expect(transaction.reason).toBeInstanceOf(SearchDeadlineError)
+      }
+      for (const resume of release) resume()
+      const settled = await Promise.allSettled(transactions)
+      /** Search indexes share the readiness read; ordinary KBs acquire their own candidate reads. */
+      expect(settled).toHaveLength(searchIndexOnly ? 1 : 18)
+      for (const transaction of settled) {
+        expect(transaction.status).toBe('rejected')
+        if (transaction.status === 'rejected')
+          expect(transaction.reason).toBeInstanceOf(SearchDeadlineError)
+      }
+      expect(dbChainMockFns.select).not.toHaveBeenCalled()
+      expect(dbChainMockFns.execute).not.toHaveBeenCalled()
     }
-    expect(dbChainMockFns.select).not.toHaveBeenCalled()
-    expect(dbChainMockFns.execute).not.toHaveBeenCalled()
-  })
+  )
 })
 
 describe('workspace search filters before ranking', () => {
@@ -943,6 +967,7 @@ describe('hydration follows ranked candidates', () => {
   }
   const params: SearchParams = {
     knowledgeBaseIds: ['org-index'],
+    searchIndexOnly: true,
     topK: 1,
     access: identity,
     accessProvider: provider,
@@ -1296,6 +1321,7 @@ describe('permitted-document planner', () => {
   }
   const params: SearchParams = {
     knowledgeBaseIds: ['org-index'],
+    searchIndexOnly: true,
     topK: 1,
     access: reader,
     accessProvider: provider,
@@ -2278,6 +2304,7 @@ describe('permitted-document planner', () => {
 
   const liveSearch = {
     knowledgeBaseIds: ['org-index'],
+    searchIndexOnly: true,
     topK: 1,
     searchMode: 'hybrid' as const,
     query: 'release',
@@ -2364,48 +2391,57 @@ describe('permitted-document planner', () => {
     expect(statements().some((query) => isPageStatement(query.sql))).toBe(false)
   })
 
-  it('excludes a denied source through its documents while the projection is unfilled', async () => {
-    queueTableRows(schemaMock.knowledgeConnector, [
-      {
-        id: 'gated-src',
-        accessMode: 'admin',
-        connectorType: 'confluence',
-        githubRepository: false,
-      },
-    ])
-    dbChainMockFns.execute.mockImplementation(async (query) => {
-      const statement = render(query).sql
-      /** The fill has not reached every row, so a denied source cannot be read off the row. */
-      if (statement.includes('AS unfilled')) return [{ unfilled: true }]
-      /** The mock renders nested fragments as parameters, so the marker is found in the whole query. */
-      const rebuilt = JSON.stringify(query).includes('/* excluded sources */')
-      if (isPageStatement(statement))
-        return JSON.stringify(render(query).params).includes('"b"')
-          ? [hit('b', 'other-src')]
-          : [hit('a', 'gated-src')]
-      if (isWalk(statement))
-        return Array.from({ length: 400 }, (_, i) => ({
-          id: i === 0 ? (rebuilt ? 'b' : 'a') : `w-${i}`,
-          distance: 0.1,
-        }))
-      return []
-    })
-    queueTableRows(schemaMock.embedding, [])
-    queueTableRows(schemaMock.embedding, [hit('b', 'other-src')])
-    const getForConnectors = vi.fn<KnowledgeAccessProvider['getForConnectors']>(async () => reader)
-    const result = await retrieveKnowledgeSearch({
-      ...liveSearch,
-      searchMode: 'vector',
-      access: reader,
-      accessProvider: { ...provider, getForConnectors },
-    })
-    expect(result.rows.map((row) => row.id)).toEqual(['b'])
-    const walks = statements().filter((query) => isWalk(query.sql))
-    expect(walks).toHaveLength(2)
-    expect(JSON.stringify(walks[0])).not.toContain('/* excluded sources */')
-    expect(JSON.stringify(walks[1])).toContain('NOT EXISTS (SELECT 1 FROM')
-    expect(JSON.stringify(walks[1])).toContain('/* excluded sources */')
-  })
+  it.each([false, true])(
+    'excludes a denied source through its documents (searchIndexOnly=%s)',
+    async (searchIndexOnly) => {
+      queueTableRows(schemaMock.knowledgeConnector, [
+        {
+          id: 'gated-src',
+          accessMode: 'admin',
+          connectorType: 'confluence',
+          githubRepository: false,
+        },
+      ])
+      dbChainMockFns.execute.mockImplementation(async (query) => {
+        const statement = render(query).sql
+        /** The fill has not reached every row, so a denied source cannot be read off the row. */
+        if (statement.includes('AS unfilled')) return [{ unfilled: true }]
+        /** The mock renders nested fragments as parameters, so the marker is found in the whole query. */
+        const rebuilt = JSON.stringify(query).includes('/* excluded sources */')
+        if (isPageStatement(statement))
+          return JSON.stringify(render(query).params).includes('"b"')
+            ? [hit('b', 'other-src')]
+            : [hit('a', 'gated-src')]
+        if (isWalk(statement))
+          return Array.from({ length: 400 }, (_, i) => ({
+            id: i === 0 ? (rebuilt ? 'b' : 'a') : `w-${i}`,
+            distance: 0.1,
+          }))
+        return []
+      })
+      queueTableRows(schemaMock.embedding, [])
+      queueTableRows(schemaMock.embedding, [hit('b', 'other-src')])
+      const getForConnectors = vi.fn<KnowledgeAccessProvider['getForConnectors']>(
+        async () => reader
+      )
+      const result = await retrieveKnowledgeSearch({
+        ...liveSearch,
+        searchIndexOnly,
+        searchMode: 'vector',
+        access: reader,
+        accessProvider: { ...provider, getForConnectors },
+      })
+      expect(result.rows.map((row) => row.id)).toEqual(['b'])
+      expect(statements().filter((query) => query.sql.includes('AS unfilled'))).toHaveLength(
+        searchIndexOnly ? 1 : 0
+      )
+      const walks = statements().filter((query) => isWalk(query.sql))
+      expect(walks).toHaveLength(2)
+      expect(JSON.stringify(walks[0])).not.toContain('/* excluded sources */')
+      expect(JSON.stringify(walks[1])).toContain('NOT EXISTS (SELECT 1 FROM')
+      expect(JSON.stringify(walks[1])).toContain('/* excluded sources */')
+    }
+  )
 
   it('hands back the unread slices of a page a denied source made it rebuild', async () => {
     queueTableRows(schemaMock.knowledgeConnector, [
@@ -2488,6 +2524,7 @@ describe('filters on a resolved scope', () => {
   }
   const params: SearchParams = {
     knowledgeBaseIds: ['org-index'],
+    searchIndexOnly: true,
     topK: 1,
     access: reader,
     accessProvider: provider,

apps/sim/lib/knowledge/search/queries.ts

@@ -359,6 +359,8 @@ export interface SearchParams {
   permitted?: PermittedDocuments
   /** Connector state resolved once per search, so no candidate re-derives it. */
   accessPlan?: SearchAccessPlan
+  /** Every searched base is a Sim Search index; ordinary KBs use document-backed pages. */
+  searchIndexOnly?: boolean
 }
 
 /** All valid tag slot keys */
@@ -1810,7 +1812,9 @@ async function selectVectorResults(params: SearchParams): Promise<SearchResult[]
         const plan = params.access.kind === 'user' ? params.accessPlan : undefined
         /** Two remembered facts, read together when neither is remembered. */
         const [filled, plannedIndexedSources] = await Promise.all([
-          isProjectionFilled('embedding_search', 'vector.projection_filled', params.budget),
+          params.searchIndexOnly === true
+            ? isProjectionFilled('embedding_search', 'vector.projection_filled', params.budget)
+            : false,
           plan?.memberSources.length ? indexedVectorSources(params.budget) : undefined,
         ])
         /**