feat(search): simplify federated sources and add scoped secrets

Author: icecrasher321

apps/sim/app/api/knowledge/connectors/member-sync/route.ts

@@ -12,6 +12,7 @@ import { resourceScopeFromOwner } from '@/lib/core/resource-scope'
 import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { connectorIndexingCondition } from '@/lib/knowledge/connectors/indexing-policy'
 import { sweepStaleMemberObservations } from '@/lib/knowledge/connectors/member-observations'
 import {
   dispatchMemberSync,
@@ -179,6 +180,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
           lte(knowledgeConnector.nextMemberSyncAt, now),
           isNull(knowledgeConnector.archivedAt),
           isNull(knowledgeConnector.deletedAt),
+          connectorIndexingCondition(),
           isNull(knowledgeBase.deletedAt)
         )
       )

apps/sim/app/api/knowledge/connectors/sync/route.ts

@@ -13,6 +13,7 @@ import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { CONTENT_ENGINE_ACCESS_MODES } from '@/lib/knowledge/connectors/access-modes'
+import { connectorIndexingCondition } from '@/lib/knowledge/connectors/indexing-policy'
 import { dispatchSync } from '@/lib/knowledge/connectors/queue'
 import {
   CONNECTOR_AUTO_DISABLED_ERROR,
@@ -315,6 +316,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
           lte(knowledgeConnector.nextSyncAt, now),
           isNull(knowledgeConnector.archivedAt),
           isNull(knowledgeConnector.deletedAt),
+          connectorIndexingCondition(),
           isNull(knowledgeBase.deletedAt)
         )
       )

apps/sim/app/api/organizations/[id]/secret-source/route.test.ts

@@ -0,0 +1,65 @@
+/** @vitest-environment node */
+import { member, organizationSecret, organizationSecretSource } from '@sim/db/schema'
+import {
+  authMockFns,
+  createMockRequest,
+  dbChainMockFns,
+  queueTableRows,
+  resetDbChainMock,
+} from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { GET, PUT } from '@/app/api/organizations/[id]/secret-source/route'
+import { GET as getSecrets, PATCH } from '@/app/api/organizations/[id]/secret-source/secrets/route'
+
+vi.mock('@/lib/permission-groups/resolve.server', () => ({
+  getUserPermissionConfigForOrganization: vi.fn().mockResolvedValue(null),
+}))
+
+const context = { params: Promise.resolve({ id: 'org' }) }
+beforeEach(() => {
+  vi.clearAllMocks()
+  resetDbChainMock()
+  authMockFns.mockGetSession.mockResolvedValue({
+    user: { id: 'actor' },
+    session: { id: 'session' },
+  })
+})
+describe('Generic Secrets HTTP boundary', () => {
+  it('requires a session before parsing a mutation', async () => {
+    authMockFns.mockGetSession.mockResolvedValue(null)
+    const response = await PATCH(createMockRequest('PATCH', { invalid: true }), context)
+    expect(response.status).toBe(401)
+    expect(dbChainMockFns.select).not.toHaveBeenCalled()
+  })
+  it('returns the source to an organization member without caching', async () => {
+    queueTableRows(member, [{ role: 'member' }])
+    queueTableRows(organizationSecretSource, [{ id: 'source', mode: 'member' }])
+    const response = await GET(createMockRequest('GET'), context)
+    expect(response.status).toBe(200)
+    expect(response.headers.get('cache-control')).toBe('private, no-store')
+    expect(await response.json()).toEqual({ source: { id: 'source', mode: 'member' } })
+  })
+  it('refuses source changes by ordinary members', async () => {
+    queueTableRows(member, [{ role: 'member' }])
+    const response = await PUT(
+      createMockRequest('PUT', { sourceId: null, mode: 'organization' }),
+      context
+    )
+    expect(response.status).toBe(403)
+    expect(dbChainMockFns.insert).not.toHaveBeenCalledWith(organizationSecretSource)
+  })
+  it('refuses the organization editor to an ordinary member before loading secrets', async () => {
+    queueTableRows(member, [{ role: 'member' }])
+    const response = await getSecrets(
+      createMockRequest(
+        'GET',
+        undefined,
+        {},
+        'http://localhost:3000/api/organizations/org/secret-source/secrets?mode=organization'
+      ),
+      context
+    )
+    expect(response.status).toBe(403)
+    expect(dbChainMockFns.from).not.toHaveBeenCalledWith(organizationSecret)
+  })
+})

apps/sim/app/api/organizations/[id]/secret-source/route.ts

@@ -0,0 +1,47 @@
+import {
+  configureSecretSourceContract,
+  getSecretSourceContract,
+  removeSecretSourceContract,
+} from '@/lib/api/contracts/organization-secrets'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import {
+  configureOrganizationSecretSource,
+  readOrganizationSecretSource,
+  removeOrganizationSecretSource,
+} from '@/lib/organization-secrets/application/use-cases'
+
+const policy = {
+  auth: internalSessionAuth,
+  rateLimit: internalRateLimits.user({ bucketName: 'organization-secret-source' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  staticResponseHeaders: { 'Cache-Control': 'private, no-store' },
+}
+
+export const GET = defineInternalJsonRoute({
+  ...policy,
+  contract: getSecretSourceContract,
+  operation: readOrganizationSecretSource.operation,
+  useCase: readOrganizationSecretSource,
+  mapInput: ({ params }) => ({ organizationId: params.id }),
+})
+
+export const PUT = defineInternalJsonRoute({
+  ...policy,
+  contract: configureSecretSourceContract,
+  operation: configureOrganizationSecretSource.operation,
+  useCase: configureOrganizationSecretSource,
+  mapInput: ({ params, body }) => ({ organizationId: params.id, ...body }),
+})
+
+export const DELETE = defineInternalJsonRoute({
+  ...policy,
+  contract: removeSecretSourceContract,
+  operation: removeOrganizationSecretSource.operation,
+  useCase: removeOrganizationSecretSource,
+  mapInput: ({ params, body }) => ({ organizationId: params.id, ...body }),
+})

apps/sim/app/api/organizations/[id]/secret-source/secrets/route.ts

@@ -0,0 +1,39 @@
+import {
+  getOrganizationSecretsContract,
+  saveOrganizationSecretsContract,
+} from '@/lib/api/contracts/organization-secrets'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+  internalSessionAuth,
+} from '@/lib/api/server/routes'
+import {
+  readOrganizationSecrets,
+  saveOrganizationSecrets,
+} from '@/lib/organization-secrets/application/use-cases'
+
+const policy = {
+  auth: internalSessionAuth,
+  rateLimit: internalRateLimits.user({ bucketName: 'organization-secrets' }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  staticResponseHeaders: { 'Cache-Control': 'private, no-store' },
+}
+
+export const GET = defineInternalJsonRoute({
+  ...policy,
+  contract: getOrganizationSecretsContract,
+  operation: readOrganizationSecrets.operation,
+  useCase: readOrganizationSecrets,
+  mapInput: ({ params, query }) => ({ organizationId: params.id, ...query }),
+})
+
+export const PATCH = defineInternalJsonRoute({
+  ...policy,
+  contract: saveOrganizationSecretsContract,
+  operation: saveOrganizationSecrets.operation,
+  useCase: saveOrganizationSecrets,
+  parseOptions: { maxBodyBytes: 2 * 1024 * 1024 },
+  mapInput: ({ params, body }) => ({ organizationId: params.id, ...body }),
+  present: () => ({ success: true as const }),
+})

apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx

@@ -977,41 +977,61 @@ describe('grouped member integrations', () => {
 describe('live integrations backend selection', () => {
   it('connects through existing OAuth enrollment without loading indexed sources', async () => {
     mocks.live = true
+    mocks.integrations.mockReturnValue({
+      data: [{ connectorType: 'google_drive', approved: true }],
+    })
     mocks.organizationAccounts.mockReturnValue({
       data: {
         credentialGroup: {
           status: 'active',
+          mcpServers: [],
           options: [
-            { id: 'drive', provider: 'google-drive', label: 'Google Drive', status: 'active' },
+            {
+              id: 'drive',
+              provider: 'google-drive',
+              label: 'Google Drive',
+              status: 'active',
+              configurationStatus: 'ready',
+            },
           ],
         },
         viewerAccounts: [],
       },
       isError: false,
     })
     await render('', <OrganizationIntegrations />)
-    expect(container.textContent).toContain('no indexing setup is needed')
     await act(async () => buttons('Connect')[0].click())
     expect(mocks.connectOrganizationAccount).toHaveBeenCalledWith(
       { organizationId: scope.organizationId, optionId: 'drive' },
       expect.any(Object)
     )
     expect(mocks.sources).not.toHaveBeenCalled()
     expect(mocks.overview).not.toHaveBeenCalled()
-    expect(mocks.integrations).not.toHaveBeenCalled()
+    expect(mocks.integrations).toHaveBeenCalledWith(scope.organizationId)
     expect(mocks.connectSearchSource).not.toHaveBeenCalled()
   })
   it('offers reconnect for an existing personal grant', async () => {
     mocks.live = true
+    mocks.integrations.mockReturnValue({ data: [{ connectorType: 'slack', approved: true }] })
     mocks.organizationAccounts.mockReturnValue({
       data: {
         credentialGroup: {
           status: 'active',
-          options: [{ id: 'slack', provider: 'slack', label: 'Slack', status: 'active' }],
+          mcpServers: [],
+          options: [
+            {
+              id: 'slack',
+              provider: 'slack',
+              label: 'Slack',
+              status: 'active',
+              configurationStatus: 'ready',
+            },
+          ],
         },
         viewerAccounts: [
           {
             credentialId: 'my-slack',
+            providerId: 'slack',
             optionId: 'slack',
             displayName: 'My Slack',
             status: 'needs_reauth',
@@ -1023,6 +1043,6 @@ describe('live integrations backend selection', () => {
     await render('', <OrganizationIntegrations />)
     await act(async () => buttons('Reconnect')[0].click())
     expect(mocks.reconnectOrganizationAccount).toHaveBeenCalledWith('my-slack', expect.any(Object))
-    expect(container.textContent).toContain('reconnect needed')
+    expect(container.textContent).toContain('Reconnect needed')
   })
 })

apps/sim/app/o/[organizationId]/integrations/integrations.tsx

@@ -33,9 +33,7 @@ export function OrganizationIntegrations({
     <OrganizationPage
       title='Integrations'
       description={
-        features.liveEnterpriseSearch
-          ? 'Search documents, messages, and code from your connected apps'
-          : 'Connect your accounts for Sim Search'
+        features.liveEnterpriseSearch ? undefined : 'Connect your accounts for Sim Search'
       }
       searchMode='expanded'
       searchPlaceholder='Search integrations'

apps/sim/app/o/[organizationId]/integrations/live-member-integrations.test.tsx

@@ -6,6 +6,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 const mocks = vi.hoisted(() => ({
   inventory: vi.fn(),
   policies: vi.fn(),
+  secrets: vi.fn(),
+  saveSecrets: vi.fn(),
   connect: vi.fn(),
   reconnect: vi.fn(),
   refetch: vi.fn(),
@@ -15,6 +17,11 @@ vi.mock('@/hooks/queries/organization-accounts', () => ({
   useConnectOrganizationAccount: () => ({ mutate: mocks.connect }),
   useReconnectPersonalOrganizationAccount: () => ({ mutate: mocks.reconnect }),
 }))
+vi.mock('@/hooks/queries/organization-secrets', () => ({
+  useOrganizationSecretSource: mocks.secrets,
+  useConfigureOrganizationSecretSource: () => ({ mutate: mocks.saveSecrets }),
+  useRemoveOrganizationSecretSource: () => ({ mutate: vi.fn() }),
+}))
 vi.mock('@/hooks/queries/search-integrations', () => ({ useSearchIntegrations: mocks.policies }))
 vi.mock('@/app/o/[organizationId]/integrations/disconnect-account-menu', () => ({
   DisconnectAccountMenu: ({ accounts }: { accounts: { displayName: string }[] }) => (
@@ -25,6 +32,7 @@ vi.mock('@/app/workspace/[workspaceId]/integrations/components/integrations-show
   IntegrationTile: () => null,
 }))
 
+import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema'
 import { LiveMemberIntegrations } from '@/app/o/[organizationId]/integrations/live-member-integrations'
 
 let root: Root
@@ -50,6 +58,7 @@ const inventory = (overrides = {}) => ({
 })
 beforeEach(() => {
   vi.clearAllMocks()
+  mocks.secrets.mockReturnValue({ data: { source: null } })
   vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true)
   mocks.inventory.mockReturnValue({ data: inventory(), refetch: mocks.refetch })
   mocks.policies.mockReturnValue({
@@ -73,6 +82,28 @@ const render = async (search = '') => {
 const button = (label: string) =>
   [...container.querySelectorAll('button')].find((button) => button.textContent === label)
 describe('live member connection states', () => {
+  it.each(['organization', 'member'] as const)(
+    'shows Generic Secrets in %s mode without OAuth sources',
+    async (mode) => {
+      mocks.policies.mockReturnValue({ data: [] })
+      mocks.inventory.mockReturnValue({ data: inventory({ credentialGroup: null }) })
+      mocks.secrets.mockReturnValue({ data: { source: { id: 'source', mode } } })
+      await render()
+      expect(container.textContent).toContain('Generic Secrets')
+      if (mode === 'member')
+        expect(container.querySelector('a')?.getAttribute('href')).toBe(
+          '/o/org/integrations/secrets'
+        )
+      else {
+        expect(container.textContent).toContain('Organization managed')
+        expect(container.querySelector('a')).toBeNull()
+      }
+      expect(button('Configure')).toBeUndefined()
+      expect(button('Connect')).toBeUndefined()
+      await render('unrelated')
+      expect(container.textContent).not.toContain('Generic Secrets')
+    }
+  )
   it('keeps GitLab organization-managed with no personal token or connect action', async () => {
     mocks.policies.mockReturnValue({ data: [{ connectorType: 'gitlab', approved: true }] })
     await render()
@@ -145,7 +176,7 @@ describe('live member connection states', () => {
   })
   it('distinguishes filtered empty, loading, and failed states', async () => {
     await render('nothing')
-    expect(container.textContent).toContain('No integrations match')
+    expect(container.textContent).toContain('No matching integrations')
     mocks.inventory.mockReturnValue({})
     await render()
     expect(container.textContent).toContain('Loading your connections')
@@ -156,4 +187,46 @@ describe('live member connection states', () => {
     await render()
     expect(container.textContent).toContain('Network unavailable')
   })
+  it('shows no empty-state copy when the organization has no sources', async () => {
+    mocks.policies.mockReturnValue({ data: [], refetch: mocks.refetch })
+    await render()
+    expect(container.textContent).toBe('')
+  })
+  it('sends admins to Credential Groups when member sign-in is not configured', async () => {
+    mocks.inventory.mockReturnValue({ data: inventory({ credentialGroup: null, canManage: true }) })
+    await render()
+    expect(container.querySelector('a')?.getAttribute('href')).toBe(
+      '/o/org/settings/connected-accounts'
+    )
+  })
+  it('explains service-account scope without hiding member connection controls', async () => {
+    mocks.policies.mockReturnValue({
+      data: [
+        {
+          connectorType: 'github',
+          approved: true,
+          policy: {
+            ...defaultLiveSearchPolicy(),
+            accessMode: 'service_account',
+            sourceId: 'source',
+          },
+        },
+      ],
+    })
+    mocks.inventory.mockReturnValue({
+      data: inventory({
+        viewerAccounts: [
+          {
+            credentialId: 'own',
+            providerId: 'github-repositories',
+            displayName: 'reader',
+            status: 'active',
+          },
+        ],
+      }),
+    })
+    await render()
+    expect(container.textContent).toContain('Selected resources you can access')
+    expect(button('Add account')).toBeDefined()
+  })
 })