improvement(knowledge): resolve live source grants once per search

- ask a live-proof source for the caller's grants once, before either leg, and only when the scope actually reads one, instead of per candidate page
- drop the per-page source exclusion and refill loop that followed from the per-page proof
- read search result metadata under the stored predicate the rows already passed
- end candidate paging on a short page whether or not the leg reorders its results
- batch the embedding_search.connector_id backfill in independently committed keyset pages, matching the other projection backfills

Author: waleedlatif1

apps/sim/app/api/v1/knowledge/search/route.test.ts

@@ -165,7 +165,7 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
         accessProvider: provider,
       })
     )
-    expect(mockGetDocumentMetadataByIds).toHaveBeenCalledWith([], access, provider)
+    expect(mockGetDocumentMetadataByIds).toHaveBeenCalledWith([], access)
   })
 
   it.each([
@@ -224,8 +224,7 @@ describe('v1 knowledge search route — per-KB embedding model', () => {
       expect(response.status).toBe(200)
       expect(mockGetDocumentMetadataByIds).toHaveBeenCalledWith(
         ['revoked-document', 'allowed-document'],
-        access,
-        provider
+        access
       )
       expect(body.data.results).toEqual(
         allDenied

apps/sim/app/api/v1/knowledge/search/route.ts

@@ -312,7 +312,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     })
 
     const documentIds = results.map((r) => r.documentId)
-    const documentMetadataMap = await getDocumentMetadataByIds(documentIds, access, accessProvider)
+    const documentMetadataMap = await getDocumentMetadataByIds(documentIds, access)
     const readableResults = results.filter((result) => documentMetadataMap[result.documentId])
 
     return NextResponse.json({

apps/sim/lib/knowledge/application/search.test.ts

@@ -825,12 +825,7 @@ describe('knowledge search application use case', () => {
       },
     })
 
-    expect(mocks.getDocumentMetadata).toHaveBeenCalledWith(
-      ['document-1'],
-      expect.anything(),
-      expect.objectContaining({ getForDocuments: expect.any(Function) }),
-      undefined
-    )
+    expect(mocks.getDocumentMetadata).toHaveBeenCalledWith(['document-1'], expect.anything())
     expect(result.results[0]).toMatchObject({
       documentName: 'guide.pdf',
       sourceUrl: 'https://example.com/guide',

apps/sim/lib/knowledge/application/search.ts

@@ -625,9 +625,7 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({
     const basicDocumentMetadata = await measureSearchStage('metadata', () =>
       getDocumentMetadataByIds(
         rows.map((row) => row.documentId),
-        access,
-        context.access,
-        input.signal
+        access
       )
     )
     const results = rows

apps/sim/lib/knowledge/search/diagnostics.ts

@@ -29,6 +29,7 @@ export type SearchStage =
   | 'defaults'
   | 'retrieval'
   | 'access_plan'
+  | 'live_source_grants'
   | 'vector.source_exact'
   | 'vector.source_walk'
   | 'permitted_documents'
@@ -38,7 +39,6 @@ export type SearchStage =
   | 'overage_billing'
   | 'tag_definitions'
   | 'metadata'
-  | 'metadata.authorization'
   | 'metadata.sql'
   | 'metadata_provenance'
   | 'activity_recording'